From pypy.commits at gmail.com  Tue Nov  1 04:50:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 01:50:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add @unwrap_spec,
 fixes a translation error too
Message-ID: <58185740.28c9c20a.d5941.d276@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88008:3e5d31c2b04e
Date: 2016-11-01 08:59 +0000
http://bitbucket.org/pypy/pypy/changeset/3e5d31c2b04e/

Log:	Add @unwrap_spec, fixes a translation error too

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -128,19 +128,6 @@
         self.index += size
         return w_res
 
-def new_unpackiter(space, w_subtype, w_struct, w_buffer):
-    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
-    w_res = space.allocate_instance(W_UnpackIter, w_subtype)
-    w_res.__init__(w_struct, buf)
-    return w_res
-
-W_UnpackIter.typedef = TypeDef("unpack_iterator",
-    __new__=interp2app(new_unpackiter),
-    __iter__=interp2app(W_UnpackIter.descr_iter),
-    __next__=interp2app(W_UnpackIter.descr_next),
-    #__length_hint__=
-)
-
 
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
@@ -185,5 +172,19 @@
     iter_unpack=interp2app(W_Struct.descr_iter_unpack),
 )
 
+ at unwrap_spec(w_struct=W_Struct)
+def new_unpackiter(space, w_subtype, w_struct, w_buffer):
+    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+    w_res = space.allocate_instance(W_UnpackIter, w_subtype)
+    w_res.__init__(w_struct, buf)
+    return w_res
+
+W_UnpackIter.typedef = TypeDef("unpack_iterator",
+    __new__=interp2app(new_unpackiter),
+    __iter__=interp2app(W_UnpackIter.descr_iter),
+    __next__=interp2app(W_UnpackIter.descr_next),
+    #__length_hint__=
+)
+
 def clearcache(space):
     """No-op on PyPy"""

From pypy.commits at gmail.com  Tue Nov  1 04:51:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 01:51:32 -0700 (PDT)
Subject: [pypy-commit] pypy missing-tp_new: Fix
Message-ID: <58185794.2636c20a.2808.867f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: missing-tp_new
Changeset: r88009:67074ec10639
Date: 2016-11-01 09:01 +0000
http://bitbucket.org/pypy/pypy/changeset/67074ec10639/

Log:	Fix

diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -30,10 +30,7 @@
 
 @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
 def slot_tp_new(space, w_type, w_args, w_kwds):
-    _, w_descr = space.lookup_in_type_where(w_type, '__new__')
-    if w_descr is None:
-        raise oefmt(space.w_RuntimeError, "cpyext: '__new__' not found")
-    # w_descr is typically a StaticMethod
+    w_impl = space.getattr(w_type, space.wrap('__new__'))
     args = Arguments(space, [w_type],
                      w_stararg=w_args, w_starstararg=w_kwds)
-    return space.call_args(space.get(w_descr, w_type), args)
+    return space.call_args(w_impl, args)

From pypy.commits at gmail.com  Tue Nov  1 05:16:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:16:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove copy-paste-modify code
Message-ID: <58185d75.49151c0a.2531f.2faf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88010:a002427e7b9c
Date: 2016-11-01 10:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a002427e7b9c/

Log:	Remove copy-paste-modify code

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -370,7 +370,7 @@
             l += 1
         return l
 
-    def visit_FunctionDef(self, func):
+    def _visit_function(self, func, function_code_generator, extra_flag):
         self.update_position(func.lineno, True)
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
@@ -385,39 +385,22 @@
         oparg = num_defaults
         oparg |= kw_default_count << 8
         oparg |= num_annotations << 16
-        code, qualname = self.sub_scope(FunctionCodeGenerator, func.name, func,
-                                        func.lineno)
+        code, qualname = self.sub_scope(function_code_generator, func.name,
+                                        func, func.lineno)
+        code.co_flags |= extra_flag
         self._make_function(code, oparg, qualname=qualname)
         # Apply decorators.
         if func.decorator_list:
             for i in range(len(func.decorator_list)):
                 self.emit_op_arg(ops.CALL_FUNCTION, 1)
         self.name_op(func.name, ast.Store)
-    
+
+    def visit_FunctionDef(self, func):
+        self._visit_function(func, FunctionCodeGenerator, 0)
+
     def visit_AsyncFunctionDef(self, func):
-        self.update_position(func.lineno, True)
-        # Load decorators first, but apply them after the function is created.
-        self.visit_sequence(func.decorator_list)
-        args = func.args
-        assert isinstance(args, ast.arguments)
-        kw_default_count = 0
-        if args.kwonlyargs:
-            kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
-        num_annotations = self._visit_annotations(func, args, func.returns)
-        num_defaults = len(args.defaults) if args.defaults is not None else 0
-        oparg = num_defaults
-        oparg |= kw_default_count << 8
-        oparg |= num_annotations << 16
-        code, qualname = self.sub_scope(AsyncFunctionCodeGenerator, func.name, func,
-                                        func.lineno)
-        code.co_flags |= consts.CO_COROUTINE
-        self._make_function(code, oparg, qualname=qualname)
-        # Apply decorators.
-        if func.decorator_list:
-            for i in range(len(func.decorator_list)):
-                self.emit_op_arg(ops.CALL_FUNCTION, 1)
-        self.name_op(func.name, ast.Store)
+        self._visit_function(func, AsyncFunctionCodeGenerator,
+                             consts.CO_COROUTINE)
 
     def visit_Lambda(self, lam):
         self.update_position(lam.lineno)
@@ -935,14 +918,22 @@
 
     def visit_With(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_withitem(wih, 0)
+        self.handle_withitem(wih, 0, is_async=False)
 
-    def handle_withitem(self, wih, pos):
+    def handle_withitem(self, wih, pos, is_async):
         body_block = self.new_block()
         cleanup = self.new_block()
         witem = wih.items[pos]
         witem.context_expr.walkabout(self)
-        self.emit_jump(ops.SETUP_WITH, cleanup)
+        if not is_async:
+            self.emit_jump(ops.SETUP_WITH, cleanup)
+        else:
+            self.emit_op(ops.BEFORE_ASYNC_WITH)
+            self.emit_op(ops.GET_AWAITABLE)
+            self.load_const(self.space.w_None)
+            self.emit_op(ops.YIELD_FROM)
+            self.emit_jump(ops.SETUP_ASYNC_WITH, cleanup)
+
         self.use_next_block(body_block)
         self.push_frame_block(F_BLOCK_FINALLY, body_block)
         if witem.optional_vars:
@@ -952,54 +943,24 @@
         if pos == len(wih.items) - 1:
             self.visit_sequence(wih.body)
         else:
-            self.handle_withitem(wih, pos + 1)
+            self.handle_withitem(wih, pos + 1, is_async=is_async)
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_FINALLY, body_block)
         self.load_const(self.space.w_None)
         self.use_next_block(cleanup)
         self.push_frame_block(F_BLOCK_FINALLY_END, cleanup)
         self.emit_op(ops.WITH_CLEANUP_START)
+        if is_async:
+            self.emit_op(ops.GET_AWAITABLE)
+            self.load_const(self.space.w_None)
+            self.emit_op(ops.YIELD_FROM)
         self.emit_op(ops.WITH_CLEANUP_FINISH)
         self.emit_op(ops.END_FINALLY)
         self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup)
 
     def visit_AsyncWith(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_asyncwithitem(wih, 0)
-
-    def handle_asyncwithitem(self, wih, pos):
-        body_block = self.new_block()
-        cleanup = self.new_block()
-        witem = wih.items[pos]
-        witem.context_expr.walkabout(self)
-        self.emit_op(ops.BEFORE_ASYNC_WITH)
-        self.emit_op(ops.GET_AWAITABLE)
-        self.load_const(self.space.w_None)
-        self.emit_op(ops.YIELD_FROM)
-        self.emit_jump(ops.SETUP_ASYNC_WITH, cleanup)
-        self.use_next_block(body_block)
-        self.push_frame_block(F_BLOCK_FINALLY, body_block)
-        if witem.optional_vars:
-            witem.optional_vars.walkabout(self)
-        else:
-            self.emit_op(ops.POP_TOP)
-        if pos == len(wih.items) - 1:
-            self.visit_sequence(wih.body)
-        else:
-            self.handle_asyncwithitem(wih, pos + 1)
-        self.emit_op(ops.POP_BLOCK)
-        self.pop_frame_block(F_BLOCK_FINALLY, body_block)
-        self.load_const(self.space.w_None)
-        self.use_next_block(cleanup)
-        self.push_frame_block(F_BLOCK_FINALLY_END, cleanup)
-        self.emit_op(ops.WITH_CLEANUP_START)
-        self.emit_op(ops.GET_AWAITABLE)
-        self.load_const(self.space.w_None)
-        self.emit_op(ops.YIELD_FROM)
-        self.emit_op(ops.WITH_CLEANUP_FINISH)
-        self.emit_op(ops.END_FINALLY)
-        self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup)
-
+        self.handle_asyncwithitem(wih, 0, is_async=True)
 
     def visit_Raise(self, rais):
         self.update_position(rais.lineno, True)

From pypy.commits at gmail.com  Tue Nov  1 05:29:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:29:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Keyword only argument default values were
 evaluated before other defaults
Message-ID: <58186075.031f1c0a.81841.d34f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88011:b0a93780397a
Date: 2016-11-01 10:28 +0100
http://bitbucket.org/pypy/pypy/changeset/b0a93780397a/

Log:	Keyword only argument default values were evaluated before other
	defaults (CPython issue #16967)

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -376,10 +376,10 @@
         self.visit_sequence(func.decorator_list)
         args = func.args
         assert isinstance(args, ast.arguments)
+        self.visit_sequence(args.defaults)
         kw_default_count = 0
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
         num_annotations = self._visit_annotations(func, args, func.returns)
         num_defaults = len(args.defaults) if args.defaults is not None else 0
         oparg = num_defaults
@@ -406,10 +406,10 @@
         self.update_position(lam.lineno)
         args = lam.args
         assert isinstance(args, ast.arguments)
+        self.visit_sequence(args.defaults)
         kw_default_count = 0
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
         default_count = len(args.defaults) if args.defaults is not None else 0
         code, qualname = self.sub_scope(
             LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,7 +38,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 80 # bump it by 16
+pypy_incremental_magic = 96 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1258,7 +1258,6 @@
             w_ann = space.newdict(strdict=True)
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
-        defaultarguments = self.popvalues(posdefaults)
         kw_defs_w = None
         if kwdefaults:
             kw_defs_w = []
@@ -1266,7 +1265,9 @@
                 w_defvalue = self.popvalue()
                 w_defname = self.popvalue()
                 kw_defs_w.append((w_defname, w_defvalue))
-        fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
+        defaultarguments = self.popvalues(posdefaults)
+        fn = function.Function(space, codeobj, self.get_w_globals(),
+                               defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -944,6 +944,19 @@
         exc = raises(ValueError, compile, mod, 'filename', 'exec')
         assert str(exc.value) == "empty targets on Delete"
 
+    def test_evaluate_argument_definition_order(self): """
+        lst = [1, 2, 3, 4]
+        def f(a=lst.pop(), b=lst.pop(), *, c=lst.pop(), d=lst.pop()):
+            return (a, b, c, d)
+        assert f('a') == ('a', 3, 2, 1), repr(f('a'))
+        assert f() == (4, 3, 2, 1), repr(f())
+        #
+        lst = [1, 2, 3, 4]
+        f = lambda a=lst.pop(), b=lst.pop(), *, c=lst.pop(), d=lst.pop(): (
+            a, b, c, d)
+        assert f('a') == ('a', 3, 2, 1), repr(f('a'))
+        assert f() == (4, 3, 2, 1), repr(f())
+        """
 
 
 class AppTestOptimizer(object):

From pypy.commits at gmail.com  Tue Nov  1 05:33:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:33:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix docstrings (lib-python test_genexps)
Message-ID: <58186150.cf9a1c0a.cc318.330e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88012:ff0dd56d2098
Date: 2016-11-01 10:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ff0dd56d2098/

Log:	Fix docstrings (lib-python test_genexps)

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -343,11 +343,11 @@
     KIND = "generator"
 
     def descr__iter__(self):
-        """x.__iter__() <==> iter(x)"""
+        """Implement iter(self)."""
         return self.space.wrap(self)
 
     def descr_next(self):
-        """x.__next__() <==> next(x)"""
+        """Implement next(self)."""
         return self.send_ex(self.space.w_None)
 
     # Results can be either an RPython list of W_Root, or it can be an

From pypy.commits at gmail.com  Tue Nov  1 05:37:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:37:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: math.inf, math.nan
Message-ID: <58186253.640ec30a.15d65.eba9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88013:53911f563c71
Date: 2016-11-01 10:36 +0100
http://bitbucket.org/pypy/pypy/changeset/53911f563c71/

Log:	math.inf, math.nan

diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py
--- a/pypy/module/math/__init__.py
+++ b/pypy/module/math/__init__.py
@@ -11,6 +11,8 @@
     interpleveldefs = {
        'e'              : 'interp_math.get(space).w_e',
        'pi'             : 'interp_math.get(space).w_pi',
+       'inf'            : 'interp_math.get(space).w_inf',
+       'nan'            : 'interp_math.get(space).w_nan',
        'pow'            : 'interp_math.pow',
        'cosh'           : 'interp_math.cosh',
        'copysign'       : 'interp_math.copysign',
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -10,6 +10,8 @@
     def __init__(self, space):
         self.w_e = space.wrap(math.e)
         self.w_pi = space.wrap(math.pi)
+        self.w_inf = space.wrap(rfloat.INFINITY)
+        self.w_nan = space.wrap(rfloat.NAN)
 def get(space):
     return space.fromcache(State)
 
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -369,3 +369,9 @@
         assert math.gcd(-4, -10) == 2
         assert math.gcd(0, -10) == 10
         assert math.gcd(0, 0) == 0
+
+    def test_inf_nan(self):
+        import math
+        assert math.isinf(math.inf)
+        assert math.inf > -math.inf
+        assert math.isnan(math.nan)

From pypy.commits at gmail.com  Tue Nov  1 05:40:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:40:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: math.gcd() argument types
Message-ID: <581862f4.0726c20a.2867.e000@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88014:61a25d2c08c5
Date: 2016-11-01 10:39 +0100
http://bitbucket.org/pypy/pypy/changeset/61a25d2c08c5/

Log:	math.gcd() argument types

diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -1,4 +1,5 @@
 import sys
+from _operator import index
 
 def factorial(x):
     """factorial(x) -> Integral
@@ -45,8 +46,8 @@
 
 def gcd(x, y):
     """greatest common divisor of x and y"""
-    x = abs(x)
-    y = abs(y)
+    x = abs(index(x))
+    y = abs(index(y))
     while x > 0:
         x, y = y % x, x
     return y
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -369,6 +369,7 @@
         assert math.gcd(-4, -10) == 2
         assert math.gcd(0, -10) == 10
         assert math.gcd(0, 0) == 0
+        raises(TypeError, math.gcd, 0, 0.0)
 
     def test_inf_nan(self):
         import math

From pypy.commits at gmail.com  Tue Nov  1 05:48:11 2016
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 01 Nov 2016 02:48:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge branch fix-struct-unpack-Q: make
 sure that we return an int whenever we can, instead of a long
Message-ID: <581864db.41d71c0a.cca75.300b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r88015:6b78c0c00c1c
Date: 2016-11-01 10:47 +0100
http://bitbucket.org/pypy/pypy/changeset/6b78c0c00c1c/

Log:	merge branch fix-struct-unpack-Q: make sure that we return an int
	whenever we can, instead of a long

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -1,3 +1,5 @@
+from rpython.rlib.rarithmetic import (r_uint, r_ulonglong, r_longlong,
+                                      maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rstring import StringBuilder
@@ -148,7 +150,20 @@
 
     @specialize.argtype(1)
     def appendobj(self, value):
-        self.result_w.append(self.space.wrap(value))
+        # CPython tries hard to return int objects whenever it can, but
+        # space.wrap returns a long if we pass a r_uint, r_ulonglong or
+        # r_longlong. So, we need special care in those cases.
+        is_unsigned = (isinstance(value, r_uint) or
+                       isinstance(value, r_ulonglong))
+        if is_unsigned and value <= maxint:
+            w_value = self.space.wrap(intmask(value))
+        elif isinstance(value, r_longlong) and -maxint-1 <= value <= maxint:
+            w_value = self.space.wrap(intmask(value))
+        else:
+            # generic type, just use space.wrap
+            w_value = self.space.wrap(value)
+        #
+        self.result_w.append(w_value)
 
     def get_pos(self):
         return self.pos
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -431,6 +431,20 @@
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 
+    def test_unpack_fits_into_int(self):
+        import sys
+        for fmt in 'ILQq':
+            # check that we return an int, if it fits
+            buf = self.struct.pack(fmt, 42)
+            val, = self.struct.unpack(fmt, buf)
+            assert val == 42
+            assert type(val) is int
+        #
+        # check that we return a long, if it doesn't fit into an int
+        buf = self.struct.pack('Q', sys.maxint+1)
+        val, = self.struct.unpack('Q', buf)
+        assert val == sys.maxint+1
+        assert type(val) is long
 
 class AppTestStructBuffer(object):
     spaceconfig = dict(usemodules=['struct', '__pypy__'])

From pypy.commits at gmail.com  Tue Nov  1 05:58:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 02:58:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: repr() for
 operator.attrgetter/itemgetter/methodcaller
Message-ID: <5818672c.49a1c20a.34196.e593@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88016:ba41422861e0
Date: 2016-11-01 10:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ba41422861e0/

Log:	repr() for operator.attrgetter/itemgetter/methodcaller

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -56,6 +56,17 @@
             for attrs in self._multi_attrs
         ])
 
+    def __repr__(self):
+        try:
+            a = repr(self._simple_attr)
+        except AttributeError:
+            try:
+                a = repr('.'.join(self._single_attr))
+            except AttributeError:
+                lst = self._multi_attrs
+                a = ', '.join([repr('.'.join(a1)) for a1 in lst])
+        return 'operator.attrgetter(%s)' % (a,)
+
 
 class itemgetter(object):
     def __init__(self, item, *items):
@@ -71,6 +82,13 @@
         else:
             return tuple([obj[i] for i in self._idx])
 
+    def __repr__(self):
+        if self._single:
+            a = repr(self._idx)
+        else:
+            a = ', '.join([repr(i) for i in self._idx])
+        return 'operator.itemgetter(%s)' % (a,)
+
 
 class methodcaller(object):
     def __init__(self, method_name, *args, **kwargs):
@@ -80,3 +98,11 @@
 
     def __call__(self, obj):
         return getattr(obj, self._method_name)(*self._args, **self._kwargs)
+
+    def __repr__(self):
+        args = [repr(self._method_name)]
+        for a in self._args:
+            args.append(repr(a))
+        for key, value in self._kwargs.items():
+            args.append('%s=%r' % (key, value))
+        return 'operator.methodcaller(%s)' % (', '.join(args),)
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -326,3 +326,23 @@
     def test_length_hint(self):
         import _operator as operator
         assert operator.length_hint([1, 2]) == 2
+
+    def test_repr_attrgetter(self):
+        import _operator as operator
+        assert repr(operator.attrgetter("foo")) == "operator.attrgetter('foo')"
+        assert repr(operator.attrgetter("foo", 'bar')) == (
+            "operator.attrgetter('foo', 'bar')")
+        assert repr(operator.attrgetter("foo.bar")) == (
+            "operator.attrgetter('foo.bar')")
+        assert repr(operator.attrgetter("foo", 'bar.baz')) == (
+            "operator.attrgetter('foo', 'bar.baz')")
+
+    def test_repr_itemgetter(self):
+        import _operator as operator
+        assert repr(operator.itemgetter(2)) == "operator.itemgetter(2)"
+        assert repr(operator.itemgetter(2, 3)) == "operator.itemgetter(2, 3)"
+
+    def test_repr_methodcaller(self):
+        import _operator as operator
+        assert repr(operator.methodcaller("foo", "bar", baz=42)) == (
+            "operator.methodcaller('foo', 'bar', baz=42)")

From pypy.commits at gmail.com  Tue Nov  1 06:00:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 03:00:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: test and fix
Message-ID: <581867ce.cf9a1c0a.cc318.3cf1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88017:63c487334766
Date: 2016-11-01 11:00 +0100
http://bitbucket.org/pypy/pypy/changeset/63c487334766/

Log:	test and fix

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -92,6 +92,8 @@
 
 class methodcaller(object):
     def __init__(self, method_name, *args, **kwargs):
+        if not isinstance(method_name, str):
+            raise TypeError("method name must be a string")
         self._method_name = method_name
         self._args = args
         self._kwargs = kwargs
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -182,6 +182,11 @@
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
+    def test_methodcaller_not_string(self):
+        import _operator as operator
+        e = raises(TypeError, operator.methodcaller, 42)
+        assert str(e.value) == "method name must be a string"
+
     def test_index(self):
         import _operator as operator
         assert operator.index(42) == 42

From pypy.commits at gmail.com  Tue Nov  1 06:15:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 03:15:30 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: modified test to do wrap
 arithmetic on integers,
 otherwise the test result within python does not wrap around
Message-ID: <58186b42.6c8ec20a.eca8f.fa04@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88019:4bacfc342956
Date: 2016-11-01 11:14 +0100
http://bitbucket.org/pypy/pypy/changeset/4bacfc342956/

Log:	modified test to do wrap arithmetic on integers, otherwise the test
	result within python does not wrap around

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -420,7 +420,7 @@
     vec_reduce = functools.partial(vec_reduce, _vec_reduce)
 
     test_vec_int_sum = vec_reduce(st.integers(min_value=-2**(64-1), max_value=2**(64-1)-1),
-                             lambda a,b: a+b, lltype.Signed)
+                             lambda a,b: lltype.intmask(lltype.intmask(a)+lltype.intmask(b)), lltype.Signed)
     test_vec_float_sum = vec_reduce(st.floats(), lambda a,b: a+b, rffi.DOUBLE)
     test_vec_float_prod = vec_reduce(st.floats(), lambda a,b: a*b, rffi.DOUBLE)
 

From pypy.commits at gmail.com  Tue Nov  1 06:15:28 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 03:15:28 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: accum reduce on integers should
 not extract both values into the same register
Message-ID: <58186b40.640ec30a.15d65.f962@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88018:7e4d2957681d
Date: 2016-11-01 10:59 +0100
http://bitbucket.org/pypy/pypy/changeset/7e4d2957681d/

Log:	accum reduce on integers should not extract both values into the
	same register

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1,16 +1,14 @@
 import sys
-import os
-import py
 
-from rpython.jit.backend.llsupport import symbolic, jitframe, rewrite
-from rpython.jit.backend.llsupport.assembler import (GuardToken, BaseAssembler, debug_bridge)
+from rpython.jit.backend.llsupport import jitframe, rewrite
+from rpython.jit.backend.llsupport.assembler import (GuardToken, BaseAssembler)
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
-from rpython.jit.metainterp.history import (Const, VOID, ConstInt)
-from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
+from rpython.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
+        Const)
 from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.rlib.rjitlog import rjitlog as jl
-from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper import rclass
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -589,6 +589,9 @@
             self._scratch_register_value = value
         self.MOV_ri(X86_64_SCRATCH_REG.value, value)
 
+    def trap(self):
+        self.INT3()
+
     def begin_reuse_scratch_register(self):
         # --NEVER CALLED (only from a specific test)--
         # Flag the beginning of a block where it is okay to reuse the value
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -10,7 +10,8 @@
     xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, AddressLoc)
 from rpython.jit.backend.llsupport.vector_ext import VectorExt
-from rpython.jit.backend.llsupport.regalloc import get_scale, TempVar, NoVariableToSpill
+from rpython.jit.backend.llsupport.regalloc import (get_scale, TempVar,
+    NoVariableToSpill)
 from rpython.jit.metainterp.resoperation import (rop, ResOperation,
         VectorOp, VectorGuardOp)
 from rpython.rlib.objectmodel import we_are_translated, always_inline
@@ -39,7 +40,13 @@
     def is_vector(self):
         return True
     def __repr__(self):
-        return "<TempVector At %s>" % (id(self),)
+        return "<TempVector at %s>" % (id(self),)
+
+class TempInt(TempVar):
+    def __init__(self):
+        self.type = INT
+    def __repr__(self):
+        return "<TempInt at %s>" % (id(self),)
 
 class X86VectorExt(VectorExt):
 
@@ -123,12 +130,14 @@
             assert isinstance(vector_loc, RegLoc)
             assert scalar_arg is not None
             orig_scalar_loc = scalar_loc
+            tmpvar = None
             if not isinstance(scalar_loc, RegLoc):
                 # scalar loc might live in memory, use scratch register and save it back later
                 if scalar_arg.type == FLOAT:
                     scalar_loc = X86_64_XMM_SCRATCH_REG
                 else:
-                    scalar_loc = X86_64_SCRATCH_REG
+                    tmpvar = TempInt()
+                    scalar_loc = regalloc.rm.try_allocate_reg(tmpvar)
                 self.mov(orig_scalar_loc, scalar_loc)
             if accum_info.accum_operation == '+':
                 self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc)
@@ -137,12 +146,13 @@
             else:
                 not_implemented("accum operator %s not implemented" %
                                             (accum_info.accum_operation)) 
+            if tmpvar:
+                regalloc.rm.possibly_free_var(tmpvar)
             if scalar_loc is not orig_scalar_loc:
                 self.mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
     def _accum_reduce_mul(self, arg, accumloc, targetloc):
-        scratchloc = X86_64_XMM_SCRATCH_REG
         self.mov(accumloc, targetloc)
         # swap the two elements
         self.mc.SHUFPD_xxi(targetloc.value, targetloc.value, 0x01)
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -172,7 +172,7 @@
             assert scalar_arg is not None
             op = accum_info.accum_operation
             self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc)
-            if scalar_loc is r. FP_SCRATCH:
+            if scalar_loc is r.FP_SCRATCH:
                 self.regalloc_mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -400,9 +400,8 @@
                 i += size
             return accum
 
-        bits = 64
         la = data.draw(st.lists(strat, min_size=10, max_size=150))
-        la = [1.0] * 10
+        #la = [1.0] * 10
         l = len(la)
 
         accum = data.draw(strat)

From pypy.commits at gmail.com  Tue Nov  1 06:17:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 03:17:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix
Message-ID: <58186bb2.c24bc20a.123fd.f184@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88020:88f5561e826f
Date: 2016-11-01 11:16 +0100
http://bitbucket.org/pypy/pypy/changeset/88f5561e826f/

Log:	Test and fix

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -254,12 +254,10 @@
         operr.normalize_exception(space)
 
         # note: w_yielded_from is always None if 'self.running'
-        w_yf = self.w_yielded_from
-        if (w_yf is not None and
+        if (self.w_yielded_from is not None and
                     operr.match(space, space.w_GeneratorExit)):
-            self.w_yielded_from = None
             try:
-                gen_close_iter(space, w_yf)
+                self._gen_close_iter(space)
             except OperationError as e:
                 return self.send_error(e)
 
@@ -270,6 +268,16 @@
                 operr.set_traceback(tb)
         return self.send_error(operr)
 
+    def _gen_close_iter(self, space):
+        assert not self.running
+        w_yf = self.w_yielded_from
+        self.w_yielded_from = None
+        self.running = True
+        try:
+            gen_close_iter(space, w_yf)
+        finally:
+            self.running = False
+
     def descr_close(self):
         """close() -> raise GeneratorExit inside generator/coroutine."""
         if self.frame is None:
@@ -279,9 +287,8 @@
         # note: w_yielded_from is always None if 'self.running'
         w_yf = self.w_yielded_from
         if w_yf is not None:
-            self.w_yielded_from = None
             try:
-                gen_close_iter(space, w_yf)
+                self._gen_close_iter(space)
             except OperationError as e:
                 operr = e
         try:
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -417,6 +417,25 @@
         assert gen.gi_yieldfrom is None
         """
 
+    def test_gi_running_in_throw_generatorexit(self): """
+        # We must force gi_running to be True on the outer generators
+        # when running an inner custom close() method.
+        class A:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                return 42
+            def close(self):
+                closed.append(gen.gi_running)
+        def g():
+            yield from A()
+        gen = g()
+        assert next(gen) == 42
+        closed = []
+        raises(GeneratorExit, gen.throw, GeneratorExit)
+        assert closed == [True]
+        """
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline

From pypy.commits at gmail.com  Tue Nov  1 06:36:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 03:36:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix
Message-ID: <58187041.43921c0a.a78e.4a3c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88021:f9dfa88ad22a
Date: 2016-11-01 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/f9dfa88ad22a/

Log:	Test and fix

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -335,8 +335,16 @@
     def descr___iter__(self, space):
         return space.wrap(self)
 
-    def descr_length(self, space):
-        return space.wrap(0 if self.remaining == -1 else self.remaining + 1)
+    def descr_length_hint(self, space):
+        # bah, there is even a CPython test that checks that this
+        # actually calls 'len_w(w_sequence)'.  Obscure.
+        res = 0
+        if self.remaining >= 0:
+            total_length = space.len_w(self.w_sequence)
+            rem_length = self.remaining + 1
+            if rem_length <= total_length:
+                res = rem_length
+        return space.wrap(res)
 
     def descr_next(self, space):
         if self.remaining >= 0:
@@ -383,7 +391,7 @@
 W_ReversedIterator.typedef = TypeDef("reversed",
     __new__         = interp2app(W_ReversedIterator.descr___new__2),
     __iter__        = interp2app(W_ReversedIterator.descr___iter__),
-    __length_hint__ = interp2app(W_ReversedIterator.descr_length),
+    __length_hint__ = interp2app(W_ReversedIterator.descr_length_hint),
     __next__        = interp2app(W_ReversedIterator.descr_next),
     __reduce__      = interp2app(W_ReversedIterator.descr___reduce__),
     __setstate__      = interp2app(W_ReversedIterator.descr___setstate__),
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -527,6 +527,25 @@
         assert list(reversed(list(reversed("hello")))) == ['h','e','l','l','o']
         raises(TypeError, reversed, reversed("hello"))
 
+    def test_reversed_length_hint(self):
+        lst = [1, 2, 3]
+        r = reversed(lst)
+        assert r.__length_hint__() == 3
+        assert next(r) == 3
+        assert r.__length_hint__() == 2
+        lst.pop()
+        assert r.__length_hint__() == 2
+        lst.pop()
+        assert r.__length_hint__() == 0
+        raises(StopIteration, next, r)
+        #
+        r = reversed(lst)
+        assert r.__length_hint__() == 1
+        assert next(r) == 1
+        assert r.__length_hint__() == 0
+        raises(StopIteration, next, r)
+        assert r.__length_hint__() == 0
+
 
 class AppTestAllAny:
     """

From pypy.commits at gmail.com  Tue Nov  1 06:57:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 03:57:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Improve error message in some cases,
 like CPython. Use '< >' around
Message-ID: <58187534.a4a9c20a.86042.0b5a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88022:ccec49a07062
Date: 2016-11-01 11:57 +0100
http://bitbucket.org/pypy/pypy/changeset/ccec49a07062/

Log:	Improve error message in some cases, like CPython. Use '< >' around
	auto-generated scope names even though these names are not really
	public.

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -232,11 +232,7 @@
 class ModuleScope(Scope):
 
     def __init__(self):
-        Scope.__init__(self, "top")
-
-    def note_await(self, await_node):
-        raise SyntaxError("'await' outside async function", await_node.lineno,
-                          await_node.col_offset)
+        Scope.__init__(self, "<top-level>")
 
 
 class FunctionScope(Scope):
@@ -270,8 +266,11 @@
             self.has_yield_inside_try = True
             
     def note_await(self, await_node):
-        raise SyntaxError("'await' outside async function", await_node.lineno,
-                          await_node.col_offset)
+        if self.name == '<genexpr>':
+            msg = "'await' expressions in comprehensions are not supported"
+        else:
+            msg = "'await' outside async function"
+        raise SyntaxError(msg, await_node.lineno, await_node.col_offset)
 
     def note_return(self, ret):
         if ret.value:
@@ -305,11 +304,9 @@
         if (self.has_free or self.child_has_free) and not self.optimized:
             raise AssertionError("unknown reason for unoptimization")
 
+
 class AsyncFunctionScope(FunctionScope):
 
-    def __init__(self, name, lineno, col_offset):
-        FunctionScope.__init__(self, name, lineno, col_offset)
-
     def note_yield(self, yield_node):
         raise SyntaxError("'yield' inside async function", yield_node.lineno,
                           yield_node.col_offset)
@@ -521,7 +518,7 @@
         assert isinstance(args, ast.arguments)
         self.visit_sequence(args.defaults)
         self.visit_kwonlydefaults(args.kw_defaults)
-        new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset)
+        new_scope = FunctionScope("<lambda>", lamb.lineno, lamb.col_offset)
         self.push_scope(new_scope, lamb)
         lamb.args.walkabout(self)
         lamb.body.walkabout(self)
@@ -531,7 +528,7 @@
         outer = comps[0]
         assert isinstance(outer, ast.comprehension)
         outer.iter.walkabout(self)
-        new_scope = FunctionScope("genexp", node.lineno, node.col_offset)
+        new_scope = FunctionScope("<genexpr>", node.lineno, node.col_offset)
         self.push_scope(new_scope, node)
         self.implicit_arg(0)
         outer.target.walkabout(self)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1132,6 +1132,15 @@
         """
         py.test.raises(SyntaxError, self.simple_test, source, None, None)
 
+    def test_error_message_1(self):
+        source = """if 1:
+        async def f():
+            {await a for a in b}
+        """
+        e = py.test.raises(SyntaxError, self.simple_test, source, None, None)
+        assert e.value.msg == (
+            "'await' expressions in comprehensions are not supported")
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -44,7 +44,7 @@
         gen_scope = mod_scope.children[0]
         assert isinstance(gen_scope, symtable.FunctionScope)
         assert not gen_scope.children
-        assert gen_scope.name == "genexp"
+        assert gen_scope.name == "<genexpr>"
         return mod_scope, gen_scope
 
     def check_unknown(self, scp, *names):
@@ -251,7 +251,7 @@
         assert len(scp.children) == 1
         lscp = scp.children[0]
         assert isinstance(lscp, symtable.FunctionScope)
-        assert lscp.name == "lambda"
+        assert lscp.name == "<lambda>"
         assert lscp.lookup("x") == symtable.SCOPE_LOCAL
         assert lscp.lookup("y") == symtable.SCOPE_GLOBAL_IMPLICIT
         scp = self.mod_scope("lambda x=a: b")

From pypy.commits at gmail.com  Tue Nov  1 07:06:41 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 04:06:41 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: simplification,
 distinct between integer and float case at update_at_exit
Message-ID: <58187741.8926c20a.3c199.0284@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88023:cd016619fb7f
Date: 2016-11-01 12:06 +0100
http://bitbucket.org/pypy/pypy/changeset/cd016619fb7f/

Log:	simplification, distinct between integer and float case at
	update_at_exit

diff --git a/rpython/jit/backend/zarch/assembler.py b/rpython/jit/backend/zarch/assembler.py
--- a/rpython/jit/backend/zarch/assembler.py
+++ b/rpython/jit/backend/zarch/assembler.py
@@ -693,11 +693,6 @@
 
         self.fixup_target_tokens(rawstart)
         self.teardown()
-        # oprofile support
-        #if self.cpu.profile_agent is not None:
-        #    name = "Loop # %s: %s" % (looptoken.number, loopname)
-        #    self.cpu.profile_agent.native_code_written(name,
-        #                                               rawstart, full_size)
         #print(hex(rawstart+looppos))
         #import pdb; pdb.set_trace()
         return AsmInfo(ops_offset, rawstart + looppos,
@@ -1006,7 +1001,7 @@
                 return
             # move from fp register to memory
             elif loc.is_stack():
-                assert loc.type == FLOAT, "target not float location"
+                assert prev_loc.type == FLOAT, "source is not a float location"
                 offset = loc.value
                 self.mc.STDY(prev_loc, l.addr(offset, r.SPP))
                 return
@@ -1600,8 +1595,7 @@
         remap_frame_layout_mixed(self, src_locations1, dst_locations1, r.SCRATCH,
                                  src_locations2, dst_locations2, r.FP_SCRATCH, WORD)
 
-        offset = self.mc.get_relative_pos()
-        self.mc.b_abs(asminfo.rawstart)
+        self.mc.b_abs(asminfo.asmaddr)
 
         rawstart = self.materialize_loop(looptoken)
         # update the guard to jump right to this custom piece of assembler
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -168,11 +168,15 @@
             scalar_arg = accum_info.getoriginal()
             orig_scalar_loc = scalar_loc
             if not scalar_loc.is_reg():
-                scalar_loc = r.FP_SCRATCH
+                if scalar_arg.type == FLOAT:
+                    scalar_loc = r.FP_SCRATCH
+                else:
+                    scalar_loc = r.SCRATCH2
+                self.regalloc_mov(orig_scalar_loc, scalar_loc)
             assert scalar_arg is not None
             op = accum_info.accum_operation
             self._accum_reduce(op, scalar_arg, vector_loc, scalar_loc)
-            if scalar_loc is r.FP_SCRATCH:
+            if scalar_loc is not orig_scalar_loc:
                 self.regalloc_mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
@@ -268,9 +272,8 @@
         else:
             assert arg.type == INT
             # store the vector onto the stack, just below the stack pointer
-            self.mc.VST(accumloc, l.addr(0, r.SP))
-            self.mc.LG(r.SCRATCH, l.addr(0, r.SP))
-            self.mc.LG(targetloc, l.addr(8, r.SP))
+            self.mc.VLGV(r.SCRATCH, accumloc, l.addr(0), l.itemsize_to_mask(8))
+            self.mc.VLGV(targetloc, accumloc, l.addr(1), l.itemsize_to_mask(8))
             if op == '+':
                 self.mc.AGR(targetloc, r.SCRATCH)
                 return

From pypy.commits at gmail.com  Tue Nov  1 07:11:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 04:11:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: missing import
Message-ID: <58187876.45ad1c0a.8d011.6866@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88024:8151b97480d6
Date: 2016-11-01 12:11 +0100
http://bitbucket.org/pypy/pypy/changeset/8151b97480d6/

Log:	missing import

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -5,7 +5,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
-        Const)
+        Const, VOID)
 from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.rlib.rjitlog import rjitlog as jl
 from rpython.rtyper.lltypesystem import lltype, rffi

From pypy.commits at gmail.com  Tue Nov  1 07:34:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 04:34:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Modules are weakrefable now
Message-ID: <58187db2.83071c0a.50b14.2ce6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88025:f8b3c91eca76
Date: 2016-11-01 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/f8b3c91eca76/

Log:	Modules are weakrefable now

diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -188,3 +188,7 @@
         excinfo = raises(AttributeError, 'nameless.does_not_exist')
         assert (excinfo.value.args[0] ==
             "module has no attribute 'does_not_exist'")
+
+    def test_weakrefable(self):
+        import weakref
+        weakref.ref(weakref)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -632,7 +632,8 @@
     __dir__ = interp2app(Module.descr_module__dir__),
     __reduce__ = interp2app(Module.descr__reduce__),
     __dict__ = GetSetProperty(descr_get_dict, cls=Module), # module dictionaries are readonly attributes
-    __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.'
+    __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.',
+    __weakref__ = make_weakref_descr(Module),
     )
 
 getset_func_doc = GetSetProperty(Function.fget_func_doc,

From pypy.commits at gmail.com  Tue Nov  1 07:41:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 04:41:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update comment
Message-ID: <58187f87.4b1d1c0a.d3cb0.62fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88026:317199b7bd96
Date: 2016-11-01 12:41 +0100
http://bitbucket.org/pypy/pypy/changeset/317199b7bd96/

Log:	Update comment

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):

From pypy.commits at gmail.com  Tue Nov  1 09:00:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 06:00:31 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: translation issue,
 moved function in file
Message-ID: <581891ef.a8c8c20a.16482.379f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88027:dd5a308d5427
Date: 2016-11-01 13:59 +0100
http://bitbucket.org/pypy/pypy/changeset/dd5a308d5427/

Log:	translation issue, moved function in file

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -43,8 +43,8 @@
         return "<TempVector at %s>" % (id(self),)
 
 class TempInt(TempVar):
-    def __init__(self):
-        self.type = INT
+    type = INT
+
     def __repr__(self):
         return "<TempInt at %s>" % (id(self),)
 
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -180,6 +180,32 @@
                 self.regalloc_mov(scalar_loc, orig_scalar_loc)
             accum_info = accum_info.next()
 
+    def _accum_reduce(self, op, arg, accumloc, targetloc):
+        # Currently the accumulator can ONLY be 64 bit float/int
+        if arg.type == FLOAT:
+            self.mc.VX(targetloc, targetloc, targetloc)
+            self.mc.VPDI(targetloc, accumloc, accumloc, permi(1,0))
+            if op == '+':
+                self.mc.VFA(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
+                return
+            elif op == '*':
+                self.mc.VFM(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
+                return
+        else:
+            assert arg.type == INT
+            # store the vector onto the stack, just below the stack pointer
+            self.mc.VLGV(r.SCRATCH, accumloc, l.addr(0), l.itemsize_to_mask(8))
+            self.mc.VLGV(targetloc, accumloc, l.addr(1), l.itemsize_to_mask(8))
+            if op == '+':
+                self.mc.AGR(targetloc, r.SCRATCH)
+                return
+            elif op == '*':
+                self.mc.MSGR(targetloc, r.SCRATCH)
+                return
+        not_implemented("reduce sum for %s not impl." % arg)
+
+
+
     def emit_vec_int_is_true(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, argloc, sizeloc = arglocs
@@ -259,30 +285,6 @@
         else:
             self.mc.VLREP(resloc, loc0, l.itemsize_to_mask(size))
 
-    def _accum_reduce(self, op, arg, accumloc, targetloc):
-        # Currently the accumulator can ONLY be 64 bit float/int
-        if arg.type == FLOAT:
-            self.mc.VPDI(targetloc, accumloc, accumloc, permi(1,0))
-            if op == '+':
-                self.mc.VFA(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
-                return
-            elif op == '*':
-                self.mc.VFM(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
-                return
-        else:
-            assert arg.type == INT
-            # store the vector onto the stack, just below the stack pointer
-            self.mc.VLGV(r.SCRATCH, accumloc, l.addr(0), l.itemsize_to_mask(8))
-            self.mc.VLGV(targetloc, accumloc, l.addr(1), l.itemsize_to_mask(8))
-            if op == '+':
-                self.mc.AGR(targetloc, r.SCRATCH)
-                return
-            elif op == '*':
-                self.mc.MSGR(targetloc, r.SCRATCH)
-                return
-        not_implemented("reduce sum for %s not impl." % arg)
-
-
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, vecloc, sourceloc, residxloc, srcidxloc, countloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -215,9 +215,6 @@
         la = data.draw(st.lists(integers, min_size=10, max_size=150))
         l = len(la)
         lb = data.draw(st.lists(integers, min_size=l, max_size=l))
-        #la = [0] * 10
-        #l = 10
-        #lb = [0] * 10
 
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
@@ -422,7 +419,8 @@
     test_vec_int_sum = vec_reduce(st.integers(min_value=-2**(64-1), max_value=2**(64-1)-1),
                              lambda a,b: lltype.intmask(lltype.intmask(a)+lltype.intmask(b)), lltype.Signed)
     test_vec_float_sum = vec_reduce(st.floats(), lambda a,b: a+b, rffi.DOUBLE)
-    test_vec_float_prod = vec_reduce(st.floats(), lambda a,b: a*b, rffi.DOUBLE)
+    test_vec_float_prod = vec_reduce(st.floats(min_value=-100, max_value=100,
+                                               allow_nan=False, allow_infinity=False), lambda a,b: a*b, rffi.DOUBLE)
 
 
     def test_constant_expand(self):

From pypy.commits at gmail.com  Tue Nov  1 09:36:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 06:36:52 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <58189a74.44941c0a.3f33e.79ce@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88029:189e9af37446
Date: 2016-11-01 14:21 +0100
http://bitbucket.org/pypy/pypy/changeset/189e9af37446/

Log:	merge default

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
diff --git a/pypy/module/cpyext/test/test_ztranslation.py b/pypy/module/cpyext/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/cpyext/test/test_ztranslation.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-
-def test_cpyext_translates():
-    checkmodule('cpyext', '_rawffi', translate_startup=False)
diff --git a/pypy/module/micronumpy/test/test_ztranslation.py b/pypy/module/micronumpy/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/micronumpy/test/test_ztranslation.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-
-def test_numpy_translates():
-    checkmodule('micronumpy')
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -1,3 +1,5 @@
+from rpython.rlib.rarithmetic import (r_uint, r_ulonglong, r_longlong,
+                                      maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rstring import StringBuilder
@@ -148,7 +150,20 @@
 
     @specialize.argtype(1)
     def appendobj(self, value):
-        self.result_w.append(self.space.wrap(value))
+        # CPython tries hard to return int objects whenever it can, but
+        # space.wrap returns a long if we pass a r_uint, r_ulonglong or
+        # r_longlong. So, we need special care in those cases.
+        is_unsigned = (isinstance(value, r_uint) or
+                       isinstance(value, r_ulonglong))
+        if is_unsigned and value <= maxint:
+            w_value = self.space.wrap(intmask(value))
+        elif isinstance(value, r_longlong) and -maxint-1 <= value <= maxint:
+            w_value = self.space.wrap(intmask(value))
+        else:
+            # generic type, just use space.wrap
+            w_value = self.space.wrap(value)
+        #
+        self.result_w.append(w_value)
 
     def get_pos(self):
         return self.pos
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -431,6 +431,20 @@
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 
+    def test_unpack_fits_into_int(self):
+        import sys
+        for fmt in 'ILQq':
+            # check that we return an int, if it fits
+            buf = self.struct.pack(fmt, 42)
+            val, = self.struct.unpack(fmt, buf)
+            assert val == 42
+            assert type(val) is int
+        #
+        # check that we return a long, if it doesn't fit into an int
+        buf = self.struct.pack('Q', sys.maxint+1)
+        val, = self.struct.unpack('Q', buf)
+        assert val == sys.maxint+1
+        assert type(val) is long
 
 class AppTestStructBuffer(object):
     spaceconfig = dict(usemodules=['struct', '__pypy__'])

From pypy.commits at gmail.com  Tue Nov  1 09:36:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 06:36:50 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: skip some vector tests where
 jit summary could not be extracted
Message-ID: <58189a72.c6bdc20a.7d720.422f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88028:7319f8c6a1d9
Date: 2016-11-01 14:16 +0100
http://bitbucket.org/pypy/pypy/changeset/7319f8c6a1d9/

Log:	skip some vector tests where jit summary could not be extracted

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -78,7 +78,7 @@
         ('sum','float', 2581, 2581, 1),
         ('prod','float', 1, 3178, 1),
         ('prod','int', 1, 3178, 1),
-        ('any','int', 1, 1239, 1),
+        ('any','int', 1, 2239, 1),
         ('any','int', 0, 4912, 0),
         ('all','int', 0, 3420, 0),
         ('all','int', 1, 6757, 1),
@@ -109,6 +109,8 @@
         vlog = self.run(main, [], vec=1)
         assert log.result == vlog.result
         assert log.result == result
+        if not log.jit_summary:
+            return
         assert log.jit_summary.vecopt_tried == 0
         assert log.jit_summary.vecopt_success == 0
         assert vlog.jit_summary.vecopt_tried > 0
@@ -186,30 +188,30 @@
         assert log.result is True
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
-        # loop.match("""
-        #     f31 = raw_load_f(i9, i29, 1, 0, descr=<ArrayF 8>)
-        #     guard_not_invalidated(descr=...)
-        #     v32 = float_ne(f31, 0.000000)
-        #     guard_true(i32, descr=...)
-        #     i36 = int_add(i24, 1)
-        #     i37 = int_add(i29, 8)
-        #     i38 = int_ge(i36, i30)
-        #     guard_false(i38, descr=...)
-        #     jump(..., descr=...)
-        #     """)
+        loop.match("""
+            f31 = raw_load_f(i9, i29, descr=<ArrayF 8>)
+            guard_not_invalidated(descr=...)
+            i32 = float_ne(f31, 0.000000)
+            guard_true(i32, descr=...)
+            i36 = int_add(i24, 1)
+            i37 = int_add(i29, 8)
+            i38 = int_ge(i36, i30)
+            guard_false(i38, descr=...)
+            jump(..., descr=...)
+            """)
         # vector version
-        assert loop.match("""
-            guard_not_invalidated(descr=...)
-            i38 = int_add(i25, 2)
-            i39 = int_ge(i38, i33)
-            guard_false(i39, descr=...)
-            v42 = vec_load_f(i9, i32, 1, 0, descr=<ArrayF 8>)
-            v43 = vec_float_ne(v42, v36)
-            f46 = vec_unpack_f(v42, 0, 1)
-            vec_guard_true(v43, descr=...)
-            i48 = int_add(i32, 16)
-            i50 = int_add(i25, 2)
-            jump(..., descr=...)""")
+        #assert loop.match("""
+        #    guard_not_invalidated(descr=...)
+        #    i38 = int_add(i25, 2)
+        #    i39 = int_ge(i38, i33)
+        #    guard_false(i39, descr=...)
+        #    v42 = vec_load_f(i9, i32, 1, 0, descr=<ArrayF 8>)
+        #    v43 = vec_float_ne(v42, v36)
+        #    f46 = vec_unpack_f(v42, 0, 1)
+        #    vec_guard_true(v43, descr=...)
+        #    i48 = int_add(i32, 16)
+        #    i50 = int_add(i25, 2)
+        #    jump(..., descr=...)""")
 
     def test_array_getitem_basic(self):
         def main():

From pypy.commits at gmail.com  Tue Nov  1 09:36:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 06:36:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-ssl: merge py3.5
Message-ID: <58189a76.cbcf1c0a.ca431.9911@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88030:89ea2e4a319d
Date: 2016-11-01 14:36 +0100
http://bitbucket.org/pypy/pypy/changeset/89ea2e4a319d/

Log:	merge py3.5

diff too long, truncating to 2000 out of 2128 lines

diff --git a/lib-python/3/test/test_copy.py b/lib-python/3/test/test_copy.py
--- a/lib-python/3/test/test_copy.py
+++ b/lib-python/3/test/test_copy.py
@@ -7,6 +7,7 @@
 from operator import le, lt, ge, gt, eq, ne
 
 import unittest
+from test import support
 
 order_comparisons = le, lt, ge, gt
 equality_comparisons = eq, ne
diff --git a/lib-python/3/test/test_long.py b/lib-python/3/test/test_long.py
--- a/lib-python/3/test/test_long.py
+++ b/lib-python/3/test/test_long.py
@@ -967,7 +967,7 @@
             self.assertIs(type(got), int)
 
         # bad second argument
-        bad_exponents = ('brian', 2.0, 0j, None)
+        bad_exponents = ('brian', 2.0, 0j)
         for e in bad_exponents:
             self.assertRaises(TypeError, round, 3, e)
 
diff --git a/lib-python/3/test/test_weakset.py b/lib-python/3/test/test_weakset.py
--- a/lib-python/3/test/test_weakset.py
+++ b/lib-python/3/test/test_weakset.py
@@ -11,6 +11,7 @@
 from collections import UserString as ustr
 import gc
 import contextlib
+from test import support
 
 
 class Foo:
diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py
--- a/lib-python/3/test/test_xml_etree.py
+++ b/lib-python/3/test/test_xml_etree.py
@@ -2085,12 +2085,14 @@
         self.assertEqual(self._ilist(doc), all_tags)
         self.assertEqual(self._ilist(doc, '*'), all_tags)
 
+    @impl_detail
     def test_copy(self):
         a = ET.Element('a')
         it = a.iter()
         with self.assertRaises(TypeError):
             copy.copy(it)
 
+    @impl_detail
     def test_pickle(self):
         a = ET.Element('a')
         it = a.iter()
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -160,7 +160,7 @@
         encoded = value.encode('utf-16')[2:]
         buf = tkffi.new("char[]", encoded)
         inbuf = tkffi.cast("Tcl_UniChar*", buf)
-        return tklib.Tcl_NewUnicodeObj(inbuf, len(encoded)/2)
+        return tklib.Tcl_NewUnicodeObj(inbuf, len(encoded)//2)
     if isinstance(value, Tcl_Obj):
         tklib.Tcl_IncrRefCount(value._value)
         return value._value
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -334,6 +334,8 @@
                         realtype, quals = self._get_type_and_quals(
                             decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
                 else:
                     raise api.CDefError("unrecognized construct", decl)
         except api.FFIError as e:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,10 +519,18 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            raise api.CDefError("%r has no values explicitly defined: "
-                                "refusing to guess which integer type it is "
-                                "meant to be (unsigned/signed, int/long)"
-                % self._get_c_name())
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -1,4 +1,5 @@
 import os
+import sys
 
 try:
     basestring
@@ -74,8 +75,13 @@
     Add py_limited_api to kwds if setuptools >= 26 is in use.
     Do not alter the setting if it already exists.
     Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+
+    CPython itself should ignore the flag in a debugging version
+    (by not listing .abi3.so in the extensions it supports), but
+    it doesn't so far, creating troubles.  That's why we check
+    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds:
+    if 'py_limited_api' not in kwds and not sys.flags.debug:
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -0,0 +1,142 @@
+============
+PyPy2.7 v5.6
+============
+
+We have released PyPy2.7 v5.6, about two months after PyPy2.7 v5.4.
+This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
+
+We continue to make incremental improvements to our C-API
+compatability layer (cpyext). We pass all but a few of the tests in the
+upstream numpy `test suite`_. 
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, and some of those
+changes have been backported to PyPy2.7 where relevant
+
+We changed ``timeit`` to now report average +- standard deviation, which is
+better than the misleading minimum value reported in CPython.
+
+XXX
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.6 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.4 released Aug 31, 2016)
+=========================================================
+
+* New features
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+  * Backport fixes to rposix on windows from py2.5
+  * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
+  * Add ``inode`` to ``scandir()`` on posix systems
+  * Support more attributes on ``super``
+  * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
+  * Restore the ability to translate with CPython
+  * Update to CFFI 1.8.4
+  * Support the new buffer protocol in cpyext and numpypy
+  * Add ``rposix.sync()``
+  * Support full-precision nanosecond times in os.stat()
+  * Add documentation about the assembler backends to RPYthon
+  * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
+    changes downstream packaging requirments
+  * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
+  * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
+    be ``str`` or ``unicode``)
+  * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
+  * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
+    relevant platforms
+  * Backport the ``'faulthandler`` module from py3.5
+  * Improve the error message when trying to call a method where the ``self``
+    parameter is missing in the definition
+  * Implement ``rposix.cpu_count``
+  * Support translation on FreeBSD running on PowerPC
+  * Implement ``__rmod__`` on ``str`` and ``unicode`` types
+  * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+
+* Bug Fixes
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Fix translation of the sandbox
+  * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+  * fix some calls to functions through window's COM interface
+  * fix minor leak when the C call to socketpair() fails
+  * make sure (-1.0 + 0j).__hash__(), (-1.0).__hash__() returns -2
+  * Fix for an issue where PyBytesResize was called on a fresh pyobj
+  * Fix bug in codewriter about passing the ``exitswitch`` variable to a call
+  * Don't crash in ``merge_if_blocks`` if the values are symbolics
+  * Issue #2325/2361: __class__ assignment between two classes with the same
+    slots
+  * Issue #2409: don't leak the file descriptor when doing ``open('some-dir')``
+  * Windows fixes around vmprof
+  * Don't use ``sprintf()`` from inside a signal handler
+  * Test and fix bug from the ``guard_not_forced_2`` branch, which didn't
+    save the floating-point register
+  * ``_numpypy.add.reduce`` returns a scalar now
+
+* Performance improvements:
+  * Improve method calls on oldstyle classes
+  * Clean and refactor code for testing cpyext to allow sharing with py3.5
+  * Refactor a building the map of reflected ops in ``_numpypy``
+  * Improve merging of virtual states in the JIT in order to avoid jumping to the
+    preamble
+  * In JIT residual calls, if the called function starts with a fast-path like
+    ``if x.foo != 0: return x.foo``, then inline the check before doing the
+     ``CALL``.
+  * Ensure ``make_inputargs`` fails properly when given arguments with type 
+    information
+  * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
+    about more easily and debugging is faster
+  * Refactor and remove dead code from ``optimizeopt``, ``resume``
+  
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.6.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -370,48 +370,24 @@
             l += 1
         return l
 
-    def visit_FunctionDef(self, func):
+    def _visit_function(self, func, function_code_generator, extra_flag):
         self.update_position(func.lineno, True)
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
         args = func.args
         assert isinstance(args, ast.arguments)
+        self.visit_sequence(args.defaults)
         kw_default_count = 0
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
         num_annotations = self._visit_annotations(func, args, func.returns)
         num_defaults = len(args.defaults) if args.defaults is not None else 0
         oparg = num_defaults
         oparg |= kw_default_count << 8
         oparg |= num_annotations << 16
-        code, qualname = self.sub_scope(FunctionCodeGenerator, func.name, func,
-                                        func.lineno)
-        self._make_function(code, oparg, qualname=qualname)
-        # Apply decorators.
-        if func.decorator_list:
-            for i in range(len(func.decorator_list)):
-                self.emit_op_arg(ops.CALL_FUNCTION, 1)
-        self.name_op(func.name, ast.Store)
-    
-    def visit_AsyncFunctionDef(self, func):
-        self.update_position(func.lineno, True)
-        # Load decorators first, but apply them after the function is created.
-        self.visit_sequence(func.decorator_list)
-        args = func.args
-        assert isinstance(args, ast.arguments)
-        kw_default_count = 0
-        if args.kwonlyargs:
-            kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
-        num_annotations = self._visit_annotations(func, args, func.returns)
-        num_defaults = len(args.defaults) if args.defaults is not None else 0
-        oparg = num_defaults
-        oparg |= kw_default_count << 8
-        oparg |= num_annotations << 16
-        code, qualname = self.sub_scope(AsyncFunctionCodeGenerator, func.name, func,
-                                        func.lineno)
-        code.co_flags |= consts.CO_COROUTINE
+        code, qualname = self.sub_scope(function_code_generator, func.name,
+                                        func, func.lineno)
+        code.co_flags |= extra_flag
         self._make_function(code, oparg, qualname=qualname)
         # Apply decorators.
         if func.decorator_list:
@@ -419,14 +395,21 @@
                 self.emit_op_arg(ops.CALL_FUNCTION, 1)
         self.name_op(func.name, ast.Store)
 
+    def visit_FunctionDef(self, func):
+        self._visit_function(func, FunctionCodeGenerator, 0)
+
+    def visit_AsyncFunctionDef(self, func):
+        self._visit_function(func, AsyncFunctionCodeGenerator,
+                             consts.CO_COROUTINE)
+
     def visit_Lambda(self, lam):
         self.update_position(lam.lineno)
         args = lam.args
         assert isinstance(args, ast.arguments)
+        self.visit_sequence(args.defaults)
         kw_default_count = 0
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        self.visit_sequence(args.defaults)
         default_count = len(args.defaults) if args.defaults is not None else 0
         code, qualname = self.sub_scope(
             LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
@@ -935,14 +918,22 @@
 
     def visit_With(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_withitem(wih, 0)
+        self.handle_withitem(wih, 0, is_async=False)
 
-    def handle_withitem(self, wih, pos):
+    def handle_withitem(self, wih, pos, is_async):
         body_block = self.new_block()
         cleanup = self.new_block()
         witem = wih.items[pos]
         witem.context_expr.walkabout(self)
-        self.emit_jump(ops.SETUP_WITH, cleanup)
+        if not is_async:
+            self.emit_jump(ops.SETUP_WITH, cleanup)
+        else:
+            self.emit_op(ops.BEFORE_ASYNC_WITH)
+            self.emit_op(ops.GET_AWAITABLE)
+            self.load_const(self.space.w_None)
+            self.emit_op(ops.YIELD_FROM)
+            self.emit_jump(ops.SETUP_ASYNC_WITH, cleanup)
+
         self.use_next_block(body_block)
         self.push_frame_block(F_BLOCK_FINALLY, body_block)
         if witem.optional_vars:
@@ -952,54 +943,24 @@
         if pos == len(wih.items) - 1:
             self.visit_sequence(wih.body)
         else:
-            self.handle_withitem(wih, pos + 1)
+            self.handle_withitem(wih, pos + 1, is_async=is_async)
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_FINALLY, body_block)
         self.load_const(self.space.w_None)
         self.use_next_block(cleanup)
         self.push_frame_block(F_BLOCK_FINALLY_END, cleanup)
         self.emit_op(ops.WITH_CLEANUP_START)
+        if is_async:
+            self.emit_op(ops.GET_AWAITABLE)
+            self.load_const(self.space.w_None)
+            self.emit_op(ops.YIELD_FROM)
         self.emit_op(ops.WITH_CLEANUP_FINISH)
         self.emit_op(ops.END_FINALLY)
         self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup)
 
     def visit_AsyncWith(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_asyncwithitem(wih, 0)
-
-    def handle_asyncwithitem(self, wih, pos):
-        body_block = self.new_block()
-        cleanup = self.new_block()
-        witem = wih.items[pos]
-        witem.context_expr.walkabout(self)
-        self.emit_op(ops.BEFORE_ASYNC_WITH)
-        self.emit_op(ops.GET_AWAITABLE)
-        self.load_const(self.space.w_None)
-        self.emit_op(ops.YIELD_FROM)
-        self.emit_jump(ops.SETUP_ASYNC_WITH, cleanup)
-        self.use_next_block(body_block)
-        self.push_frame_block(F_BLOCK_FINALLY, body_block)
-        if witem.optional_vars:
-            witem.optional_vars.walkabout(self)
-        else:
-            self.emit_op(ops.POP_TOP)
-        if pos == len(wih.items) - 1:
-            self.visit_sequence(wih.body)
-        else:
-            self.handle_asyncwithitem(wih, pos + 1)
-        self.emit_op(ops.POP_BLOCK)
-        self.pop_frame_block(F_BLOCK_FINALLY, body_block)
-        self.load_const(self.space.w_None)
-        self.use_next_block(cleanup)
-        self.push_frame_block(F_BLOCK_FINALLY_END, cleanup)
-        self.emit_op(ops.WITH_CLEANUP_START)
-        self.emit_op(ops.GET_AWAITABLE)
-        self.load_const(self.space.w_None)
-        self.emit_op(ops.YIELD_FROM)
-        self.emit_op(ops.WITH_CLEANUP_FINISH)
-        self.emit_op(ops.END_FINALLY)
-        self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup)
-
+        self.handle_asyncwithitem(wih, 0, is_async=True)
 
     def visit_Raise(self, rais):
         self.update_position(rais.lineno, True)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -232,11 +232,7 @@
 class ModuleScope(Scope):
 
     def __init__(self):
-        Scope.__init__(self, "top")
-
-    def note_await(self, await_node):
-        raise SyntaxError("'await' outside async function", await_node.lineno,
-                          await_node.col_offset)
+        Scope.__init__(self, "<top-level>")
 
 
 class FunctionScope(Scope):
@@ -270,8 +266,11 @@
             self.has_yield_inside_try = True
             
     def note_await(self, await_node):
-        raise SyntaxError("'await' outside async function", await_node.lineno,
-                          await_node.col_offset)
+        if self.name == '<genexpr>':
+            msg = "'await' expressions in comprehensions are not supported"
+        else:
+            msg = "'await' outside async function"
+        raise SyntaxError(msg, await_node.lineno, await_node.col_offset)
 
     def note_return(self, ret):
         if ret.value:
@@ -305,11 +304,9 @@
         if (self.has_free or self.child_has_free) and not self.optimized:
             raise AssertionError("unknown reason for unoptimization")
 
+
 class AsyncFunctionScope(FunctionScope):
 
-    def __init__(self, name, lineno, col_offset):
-        FunctionScope.__init__(self, name, lineno, col_offset)
-
     def note_yield(self, yield_node):
         raise SyntaxError("'yield' inside async function", yield_node.lineno,
                           yield_node.col_offset)
@@ -521,7 +518,7 @@
         assert isinstance(args, ast.arguments)
         self.visit_sequence(args.defaults)
         self.visit_kwonlydefaults(args.kw_defaults)
-        new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset)
+        new_scope = FunctionScope("<lambda>", lamb.lineno, lamb.col_offset)
         self.push_scope(new_scope, lamb)
         lamb.args.walkabout(self)
         lamb.body.walkabout(self)
@@ -531,7 +528,7 @@
         outer = comps[0]
         assert isinstance(outer, ast.comprehension)
         outer.iter.walkabout(self)
-        new_scope = FunctionScope("genexp", node.lineno, node.col_offset)
+        new_scope = FunctionScope("<genexpr>", node.lineno, node.col_offset)
         self.push_scope(new_scope, node)
         self.implicit_arg(0)
         outer.target.walkabout(self)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1132,6 +1132,15 @@
         """
         py.test.raises(SyntaxError, self.simple_test, source, None, None)
 
+    def test_error_message_1(self):
+        source = """if 1:
+        async def f():
+            {await a for a in b}
+        """
+        e = py.test.raises(SyntaxError, self.simple_test, source, None, None)
+        assert e.value.msg == (
+            "'await' expressions in comprehensions are not supported")
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -44,7 +44,7 @@
         gen_scope = mod_scope.children[0]
         assert isinstance(gen_scope, symtable.FunctionScope)
         assert not gen_scope.children
-        assert gen_scope.name == "genexp"
+        assert gen_scope.name == "<genexpr>"
         return mod_scope, gen_scope
 
     def check_unknown(self, scp, *names):
@@ -251,7 +251,7 @@
         assert len(scp.children) == 1
         lscp = scp.children[0]
         assert isinstance(lscp, symtable.FunctionScope)
-        assert lscp.name == "lambda"
+        assert lscp.name == "<lambda>"
         assert lscp.lookup("x") == symtable.SCOPE_LOCAL
         assert lscp.lookup("y") == symtable.SCOPE_GLOBAL_IMPLICIT
         scp = self.mod_scope("lambda x=a: b")
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1582,7 +1582,7 @@
         from rpython.rlib import rstring
         result = self.str_w(w_obj)
         if '\x00' in result:
-            raise oefmt(self.w_TypeError,
+            raise oefmt(self.w_ValueError,
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
@@ -1591,7 +1591,7 @@
         from rpython.rlib import rstring
         result = self.bytes_w(w_obj)
         if '\x00' in result:
-            raise oefmt(self.w_TypeError,
+            raise oefmt(self.w_ValueError,
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
@@ -1637,7 +1637,7 @@
         from rpython.rlib import rstring
         result = w_obj.unicode_w(self)
         if u'\x00' in result:
-            raise oefmt(self.w_TypeError,
+            raise oefmt(self.w_ValueError,
                         "argument must be a unicode string without NUL "
                         "characters")
         return rstring.assert_str0(result)
@@ -1671,7 +1671,7 @@
             w_obj = self.fsencode(w_obj)
         result = self.bufferstr_w(w_obj, self.BUF_FULL_RO)
         if '\x00' in result:
-            raise oefmt(self.w_TypeError,
+            raise oefmt(self.w_ValueError,
                         "argument must be a string without NUL characters")
         return rstring.assert_str0(result)
 
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -254,12 +254,10 @@
         operr.normalize_exception(space)
 
         # note: w_yielded_from is always None if 'self.running'
-        w_yf = self.w_yielded_from
-        if (w_yf is not None and
+        if (self.w_yielded_from is not None and
                     operr.match(space, space.w_GeneratorExit)):
-            self.w_yielded_from = None
             try:
-                gen_close_iter(space, w_yf)
+                self._gen_close_iter(space)
             except OperationError as e:
                 return self.send_error(e)
 
@@ -270,6 +268,16 @@
                 operr.set_traceback(tb)
         return self.send_error(operr)
 
+    def _gen_close_iter(self, space):
+        assert not self.running
+        w_yf = self.w_yielded_from
+        self.w_yielded_from = None
+        self.running = True
+        try:
+            gen_close_iter(space, w_yf)
+        finally:
+            self.running = False
+
     def descr_close(self):
         """close() -> raise GeneratorExit inside generator/coroutine."""
         if self.frame is None:
@@ -279,9 +287,8 @@
         # note: w_yielded_from is always None if 'self.running'
         w_yf = self.w_yielded_from
         if w_yf is not None:
-            self.w_yielded_from = None
             try:
-                gen_close_iter(space, w_yf)
+                self._gen_close_iter(space)
             except OperationError as e:
                 operr = e
         try:
@@ -343,11 +350,11 @@
     KIND = "generator"
 
     def descr__iter__(self):
-        """x.__iter__() <==> iter(x)"""
+        """Implement iter(self)."""
         return self.space.wrap(self)
 
     def descr_next(self):
-        """x.__next__() <==> next(x)"""
+        """Implement next(self)."""
         return self.send_ex(self.space.w_None)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -147,6 +147,21 @@
             __file__ = space.unicode_w(space.repr(w___file__))
             return space.wrap(u"<module %s from %s>" % (name, __file__))
 
+    def descr_getattribute(self, space, w_attr):
+        from pypy.objspace.descroperation import object_getattribute
+        try:
+            return space.call_function(object_getattribute(space), self, w_attr)
+        except OperationError as e:
+            if not e.match(space, space.w_AttributeError):
+                raise
+            w_name = space.finditem(self.w_dict, space.wrap('__name__'))
+            if w_name is None:
+                raise oefmt(space.w_AttributeError,
+                    "module has no attribute %R", w_attr)
+            else:
+                raise oefmt(space.w_AttributeError,
+                    "module %R has no attribute %R", w_name, w_attr)
+
     def descr_module__dir__(self, space):
         w_dict = space.getattr(self, space.wrap('__dict__'))
         if not space.isinstance_w(w_dict, space.w_dict):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,7 +38,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 80 # bump it by 16
+pypy_incremental_magic = 96 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1258,7 +1258,6 @@
             w_ann = space.newdict(strdict=True)
             for i in range(len(names_w) - 1, -1, -1):
                 space.setitem(w_ann, names_w[i], self.popvalue())
-        defaultarguments = self.popvalues(posdefaults)
         kw_defs_w = None
         if kwdefaults:
             kw_defs_w = []
@@ -1266,7 +1265,9 @@
                 w_defvalue = self.popvalue()
                 w_defname = self.popvalue()
                 kw_defs_w.append((w_defname, w_defvalue))
-        fn = function.Function(space, codeobj, self.get_w_globals(), defaultarguments,
+        defaultarguments = self.popvalues(posdefaults)
+        fn = function.Function(space, codeobj, self.get_w_globals(),
+                               defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -944,6 +944,19 @@
         exc = raises(ValueError, compile, mod, 'filename', 'exec')
         assert str(exc.value) == "empty targets on Delete"
 
+    def test_evaluate_argument_definition_order(self): """
+        lst = [1, 2, 3, 4]
+        def f(a=lst.pop(), b=lst.pop(), *, c=lst.pop(), d=lst.pop()):
+            return (a, b, c, d)
+        assert f('a') == ('a', 3, 2, 1), repr(f('a'))
+        assert f() == (4, 3, 2, 1), repr(f())
+        #
+        lst = [1, 2, 3, 4]
+        f = lambda a=lst.pop(), b=lst.pop(), *, c=lst.pop(), d=lst.pop(): (
+            a, b, c, d)
+        assert f('a') == ('a', 3, 2, 1), repr(f('a'))
+        assert f() == (4, 3, 2, 1), repr(f())
+        """
 
 
 class AppTestOptimizer(object):
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -417,6 +417,25 @@
         assert gen.gi_yieldfrom is None
         """
 
+    def test_gi_running_in_throw_generatorexit(self): """
+        # We must force gi_running to be True on the outer generators
+        # when running an inner custom close() method.
+        class A:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                return 42
+            def close(self):
+                closed.append(gen.gi_running)
+        def g():
+            yield from A()
+        gen = g()
+        assert next(gen) == 42
+        closed = []
+        raises(GeneratorExit, gen.throw, GeneratorExit)
+        assert closed == [True]
+        """
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.module import Module
 
-class TestModule: 
+class TestModule:
     def test_name(self, space):
         w = space.wrap
         m = Module(space, space.wrap('m'))
@@ -31,7 +31,7 @@
         py.test.raises(OperationError, space.getattr, w(m), w('__file__'))
 
 
-class AppTest_ModuleObject: 
+class AppTest_ModuleObject:
     def test_attr(self):
         m = __import__('builtins')
         m.x = 15
@@ -66,7 +66,7 @@
             skip("need PyPy for _pypy_interact")
         r = repr(sys)
         assert r == "<module 'sys' (built-in)>"
-        
+
         import _pypy_interact # known to be in lib_pypy
         r = repr(_pypy_interact)
         assert (r.startswith("<module '_pypy_interact' from ") and
@@ -123,7 +123,7 @@
         test_module.__loader__ = CustomLoaderWithRaisingRepr
         mod_repr = repr(test_module)
 
-        # The module has no __file__ attribute, so the repr should use 
+        # The module has no __file__ attribute, so the repr should use
         # the loader and name
         loader_repr = repr(test_module.__loader__)
         expected_repr = "<module 'test_module' ({})>".format(loader_repr)
@@ -174,3 +174,21 @@
         m = type(sys)('&#26085;&#26412;')
         assert m.__name__ == '&#26085;&#26412;'
         assert repr(m).startswith("<module '&#26085;&#26412;'")
+
+    def test_AttributeError_message(self):
+        import sys
+        test_module = type(sys)("test_module", "doc")
+        excinfo = raises(AttributeError, 'test_module.does_not_exist')
+        assert (excinfo.value.args[0] ==
+            "module 'test_module' has no attribute 'does_not_exist'")
+
+        nameless = type(sys)("nameless", "doc")
+        del nameless.__name__
+        assert not hasattr(nameless, '__name__')
+        excinfo = raises(AttributeError, 'nameless.does_not_exist')
+        assert (excinfo.value.args[0] ==
+            "module has no attribute 'does_not_exist'")
+
+    def test_weakrefable(self):
+        import weakref
+        weakref.ref(weakref)
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -210,9 +210,9 @@
         space = self.space
         w = space.wrap
         assert space.str0_w(w("123")) == "123"
-        exc = space.raises_w(space.w_TypeError, space.str0_w, w("123\x004"))
+        exc = space.raises_w(space.w_ValueError, space.str0_w, w("123\x004"))
         assert space.unicode0_w(w(u"123")) == u"123"
-        exc = space.raises_w(space.w_TypeError, space.unicode0_w, w(u"123\x004"))
+        exc = space.raises_w(space.w_ValueError, space.unicode0_w, w(u"123\x004"))
 
     def test_identifier_w(self):
         space = self.space
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -628,10 +628,12 @@
     __new__ = interp2app(Module.descr_module__new__.im_func),
     __init__ = interp2app(Module.descr_module__init__),
     __repr__ = interp2app(Module.descr_module__repr__),
+    __getattribute__=interp2app(Module.descr_getattribute),
     __dir__ = interp2app(Module.descr_module__dir__),
     __reduce__ = interp2app(Module.descr__reduce__),
     __dict__ = GetSetProperty(descr_get_dict, cls=Module), # module dictionaries are readonly attributes
-    __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.'
+    __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.',
+    __weakref__ = make_weakref_descr(Module),
     )
 
 getset_func_doc = GetSetProperty(Function.fget_func_doc,
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -335,8 +335,16 @@
     def descr___iter__(self, space):
         return space.wrap(self)
 
-    def descr_length(self, space):
-        return space.wrap(0 if self.remaining == -1 else self.remaining + 1)
+    def descr_length_hint(self, space):
+        # bah, there is even a CPython test that checks that this
+        # actually calls 'len_w(w_sequence)'.  Obscure.
+        res = 0
+        if self.remaining >= 0:
+            total_length = space.len_w(self.w_sequence)
+            rem_length = self.remaining + 1
+            if rem_length <= total_length:
+                res = rem_length
+        return space.wrap(res)
 
     def descr_next(self, space):
         if self.remaining >= 0:
@@ -383,7 +391,7 @@
 W_ReversedIterator.typedef = TypeDef("reversed",
     __new__         = interp2app(W_ReversedIterator.descr___new__2),
     __iter__        = interp2app(W_ReversedIterator.descr___iter__),
-    __length_hint__ = interp2app(W_ReversedIterator.descr_length),
+    __length_hint__ = interp2app(W_ReversedIterator.descr_length_hint),
     __next__        = interp2app(W_ReversedIterator.descr_next),
     __reduce__      = interp2app(W_ReversedIterator.descr___reduce__),
     __setstate__      = interp2app(W_ReversedIterator.descr___setstate__),
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -527,6 +527,25 @@
         assert list(reversed(list(reversed("hello")))) == ['h','e','l','l','o']
         raises(TypeError, reversed, reversed("hello"))
 
+    def test_reversed_length_hint(self):
+        lst = [1, 2, 3]
+        r = reversed(lst)
+        assert r.__length_hint__() == 3
+        assert next(r) == 3
+        assert r.__length_hint__() == 2
+        lst.pop()
+        assert r.__length_hint__() == 2
+        lst.pop()
+        assert r.__length_hint__() == 0
+        raises(StopIteration, next, r)
+        #
+        r = reversed(lst)
+        assert r.__length_hint__() == 1
+        assert next(r) == 1
+        assert r.__length_hint__() == 0
+        raises(StopIteration, next, r)
+        assert r.__length_hint__() == 0
+
 
 class AppTestAllAny:
     """
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -3,6 +3,7 @@
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
+from pypy.module._cffi_backend import ctypestruct
 
 from rpython.rlib.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
@@ -89,7 +90,12 @@
     ctype = w_cdata.ctype
     if isinstance(ctype, ctypeptr.W_CTypePointer):
         if size < 0:
-            size = ctype.ctitem.size
+            structobj = w_cdata.get_structobj()
+            if (structobj is not None and
+                isinstance(structobj.ctype, ctypestruct.W_CTypeStructOrUnion)):
+                size = structobj._sizeof()
+            if size < 0:
+                size = ctype.ctitem.size
     elif isinstance(ctype, ctypearray.W_CTypeArray):
         if size < 0:
             size = w_cdata._sizeof()
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -329,7 +329,7 @@
     def getattr(self, w_attr):
         cfield = self.getcfield(w_attr)
         with self as ptr:
-            w_res = cfield.read(ptr)
+            w_res = cfield.read(ptr, self)
         return w_res
 
     def setattr(self, w_attr, w_value):
@@ -432,6 +432,9 @@
         lst = ct.cdata_dir()
         return space.newlist([space.wrap(s) for s in lst])
 
+    def get_structobj(self):
+        return None
+
 
 class W_CDataMem(W_CData):
     """This is used only by the results of cffi.cast('int', x)
@@ -453,28 +456,36 @@
     by newp().  They create and free their own memory according to an
     allocator."""
 
-    # the 'length' is either >= 0 for arrays, or -1 for pointers.
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
+    # the 'allocated_length' is >= 0 for arrays; for var-sized
+    # structures it is the total size in bytes; otherwise it is -1.
+    _attrs_ = ['allocated_length']
+    _immutable_fields_ = ['allocated_length']
 
     def __init__(self, space, cdata, ctype, length=-1):
         W_CData.__init__(self, space, cdata, ctype)
-        self.length = length
+        self.allocated_length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
     def _sizeof(self):
         ctype = self.ctype
-        if self.length >= 0:
+        if self.allocated_length >= 0:
             from pypy.module._cffi_backend import ctypearray
-            assert isinstance(ctype, ctypearray.W_CTypeArray)
-            return self.length * ctype.ctitem.size
+            if isinstance(ctype, ctypearray.W_CTypeArray):
+                return self.allocated_length * ctype.ctitem.size
+            else:
+                return self.allocated_length    # var-sized struct size
         else:
             return ctype.size
 
     def get_array_length(self):
-        return self.length
+        from pypy.module._cffi_backend import ctypearray
+        assert isinstance(self.ctype, ctypearray.W_CTypeArray)
+        return self.allocated_length
+
+    def get_structobj(self):
+        return self
 
 
 class W_CDataNewStd(W_CDataNewOwning):
@@ -508,12 +519,19 @@
         self.structobj = structobj
 
     def _repr_extra(self):
-        return self._repr_extra_owning()
+        return self.structobj._repr_extra_owning()
 
     def _do_getitem(self, ctype, i):
         assert i == 0
         return self.structobj
 
+    def get_structobj(self):
+        structobj = self.structobj
+        if isinstance(structobj, W_CDataNewOwning):
+            return structobj
+        else:
+            return None
+
 
 class W_CDataSliced(W_CData):
     """Subclass with an explicit length, for slices."""
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -213,13 +213,16 @@
             # a W_CDataPtrToStruct object which has a strong reference
             # to a W_CDataNewOwning that really contains the structure.
             #
-            if not space.is_w(w_init, space.w_None):
-                ctitem.force_lazy_struct()
-                if ctitem._with_var_array:
+            varsize_length = -1
+            ctitem.force_lazy_struct()
+            if ctitem._with_var_array:
+                if not space.is_w(w_init, space.w_None):
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
+                varsize_length = datasize
             #
-            cdatastruct = allocator.allocate(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem,
+                                             length=varsize_length)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -210,7 +210,7 @@
         return W_CField(self.ctype, offset + self.offset,
                         self.bitshift, self.bitsize, self.flags | fflags)
 
-    def read(self, cdata):
+    def read(self, cdata, w_cdata):
         cdata = rffi.ptradd(cdata, self.offset)
         if self.bitshift == self.BS_REGULAR:
             return self.ctype.convert_to_object(cdata)
@@ -218,6 +218,14 @@
             from pypy.module._cffi_backend import ctypearray
             ctype = self.ctype
             assert isinstance(ctype, ctypearray.W_CTypeArray)
+            structobj = w_cdata.get_structobj()
+            if structobj is not None:
+                # variable-length array
+                size = structobj.allocated_length - self.offset
+                if size >= 0:
+                    arraylen = size // ctype.ctitem.size
+                    return cdataobj.W_CDataSliced(ctype.space, cdata, ctype,
+                                                  arraylen)
             return cdataobj.W_CData(ctype.space, cdata, ctype.ctptr)
         else:
             return self.convert_bitfield_to_object(cdata)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -353,7 +353,7 @@
         if fbitsize < 0:
             # not a bitfield: common case
 
-            if isinstance(ftype, ctypearray.W_CTypeArray) and ftype.length==0:
+            if isinstance(ftype, ctypearray.W_CTypeArray) and ftype.length<=0:
                 bs_flag = ctypestruct.W_CField.BS_EMPTY_ARRAY
             else:
                 bs_flag = ctypestruct.W_CField.BS_REGULAR
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3161,17 +3161,19 @@
     assert d[1][0] == 'y'
     assert d[1][1].type is BArray
     assert d[1][1].offset == size_of_int()
-    assert d[1][1].bitshift == -1
+    assert d[1][1].bitshift == -2
     assert d[1][1].bitsize == -1
     #
     p = newp(new_pointer_type(BStruct))
     p.x = 42
     assert p.x == 42
-    assert typeof(p.y) is BIntP
+    assert typeof(p.y) is BArray
+    assert len(p.y) == 0
     assert p.y == cast(BIntP, p) + 1
     #
     p = newp(new_pointer_type(BStruct), [100])
     assert p.x == 100
+    assert len(p.y) == 0
     #
     # Tests for
     #    ffi.new("struct_with_var_array *", [field.., [the_array_items..]])
@@ -3186,6 +3188,10 @@
             p.y[0] = 200
             assert p.y[2] == 0
             p.y[2] = 400
+        assert len(p.y) == 3
+        assert len(p[0].y) == 3
+        assert len(buffer(p)) == sizeof(BInt) * 4
+        assert sizeof(p[0]) == sizeof(BInt) * 4
         plist.append(p)
     for i in range(20):
         p = plist[i]
@@ -3193,13 +3199,31 @@
         assert p.y[0] == 200
         assert p.y[1] == i
         assert p.y[2] == 400
-        assert list(p.y[0:3]) == [200, i, 400]
+        assert list(p.y) == [200, i, 400]
     #
     # the following assignment works, as it normally would, for any array field
-    p.y = [500, 600]
-    assert list(p.y[0:3]) == [500, 600, 400]
+    p.y = [501, 601]
+    assert list(p.y) == [501, 601, 400]
+    p[0].y = [500, 600]
+    assert list(p[0].y) == [500, 600, 400]
+    assert repr(p) == "<cdata 'foo *' owning %d bytes>" % (
+        sizeof(BStruct) + 3 * sizeof(BInt),)
+    assert repr(p[0]) == "<cdata 'foo' owning %d bytes>" % (
+        sizeof(BStruct) + 3 * sizeof(BInt),)
+    assert sizeof(p[0]) == sizeof(BStruct) + 3 * sizeof(BInt)
+    #
+    # from a non-owning pointer, we can't get the length
+    q = cast(new_pointer_type(BStruct), p)
+    assert q.y[0] == 500
+    assert q[0].y[0] == 500
+    py.test.raises(TypeError, len, q.y)
+    py.test.raises(TypeError, len, q[0].y)
+    assert typeof(q.y) is BIntP
+    assert typeof(q[0].y) is BIntP
+    assert sizeof(q[0]) == sizeof(BStruct)
     #
     # error cases
+    py.test.raises(IndexError, "p.y[4]")
     py.test.raises(TypeError, "p.y = cast(BIntP, 0)")
     py.test.raises(TypeError, "p.y = 15")
     py.test.raises(TypeError, "p.y = None")
@@ -3264,6 +3288,33 @@
         assert p.x[5] == 60
         assert p.x[6] == 70
 
+def test_struct_array_not_aligned():
+    # struct a { int x; char y; char z[]; };
+    # ends up of size 8, but 'z' is at offset 5
+    BChar = new_primitive_type("char")
+    BInt = new_primitive_type("int")
+    BCharP = new_pointer_type(BChar)
+    BArray = new_array_type(BCharP, None)
+    BStruct = new_struct_type("foo")
+    complete_struct_or_union(BStruct, [('x', BInt),
+                                       ('y', BChar),
+                                       ('z', BArray)])
+    assert sizeof(BStruct) == 2 * size_of_int()
+    def offsetof(BType, fieldname):
+        return typeoffsetof(BType, fieldname)[1]
+    base = offsetof(BStruct, 'z')
+    assert base == size_of_int() + 1
+    #
+    p = newp(new_pointer_type(BStruct), {'z': 3})
+    assert sizeof(p[0]) == base + 3
+    q = newp(new_pointer_type(BStruct), {'z': size_of_int()})
+    assert sizeof(q) == size_of_ptr()
+    assert sizeof(q[0]) == base + size_of_int()
+    assert len(p.z) == 3
+    assert len(p[0].z) == 3
+    assert len(q.z) == size_of_int()
+    assert len(q[0].z) == size_of_int()
+
 def test_ass_slice():
     BChar = new_primitive_type("char")
     BArray = new_array_type(new_pointer_type(BChar), None)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -408,11 +408,14 @@
             'test_misdeclared_field_1',
             "struct foo_s { int a[6]; };")
         assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
-        p = ffi.new("struct foo_s *")
-        # lazily build the fields and boom:
-        e = raises(ffi.error, getattr, p, "a")
-        assert str(e.value).startswith("struct foo_s: wrong size for field 'a' "
-                                       "(cdef says 20, but C compiler says 24)")
+        try:
+            # lazily build the fields and boom:
+            p = ffi.new("struct foo_s *")
+            p.a
+            assert False, "should have raised"
+        except ffi.error as e:
+            assert str(e).startswith("struct foo_s: wrong size for field 'a' "
+                                     "(cdef says 20, but C compiler says 24)")
 
     def test_open_array_in_struct(self):
         ffi, lib = self.prepare(
@@ -420,8 +423,10 @@
             'test_open_array_in_struct',
             "struct foo_s { int b; int a[]; };")
         assert ffi.sizeof("struct foo_s") == 4
-        p = ffi.new("struct foo_s *", [5, [10, 20, 30]])
+        p = ffi.new("struct foo_s *", [5, [10, 20, 30, 40]])
         assert p.a[2] == 30
+        assert ffi.sizeof(p) == ffi.sizeof("void *")
+        assert ffi.sizeof(p[0]) == 5 * ffi.sizeof("int")
 
     def test_math_sin_type(self):
         ffi, lib = self.prepare(
@@ -954,6 +959,7 @@
             "struct foo_s { int x; int a[5][8]; int y; };")
         assert ffi.sizeof('struct foo_s') == 42 * ffi.sizeof('int')
         s = ffi.new("struct foo_s *")
+        assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
         assert ffi.sizeof(s.a) == 40 * ffi.sizeof('int')
         assert s.a[4][7] == 0
         raises(IndexError, 's.a[4][8]')
@@ -961,6 +967,18 @@
         assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
         assert ffi.typeof(s.a[0]) == ffi.typeof("int[8]")
 
+    def test_struct_array_guess_length_3(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int a[][...]; };",
+            'test_struct_array_guess_length_3',
+            "struct foo_s { int x; int a[5][7]; int y; };")
+        assert ffi.sizeof('struct foo_s') == 37 * ffi.sizeof('int')
+        s = ffi.new("struct foo_s *")
+        assert ffi.typeof(s.a) == ffi.typeof("int[][7]")
+        assert s.a[4][6] == 0
+        raises(IndexError, 's.a[4][7]')
+        assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]")
+
     def test_global_var_array_2(self):
         ffi, lib = self.prepare(
             "int a[...][...];",
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -35,6 +35,9 @@
         rwin32.BOOL,
         save_err=rffi.RFFI_SAVE_LASTERROR)
 
+    def sem_unlink(name):
+        pass
+
 else:
     from rpython.rlib import rposix
 
@@ -325,12 +328,7 @@
 else:
     def create_semaphore(space, name, val, max):
         sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
-        try:
-            sem_unlink(name)
-        except OSError:
-            pass
-        else:
-            rgc.add_memory_pressure(SEM_T_SIZE)
+        rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
     def delete_semaphore(handle):
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -22,8 +22,10 @@
     @py.test.mark.skipif("sys.platform == 'win32'")
     def test_sem_unlink(self):
         from _multiprocessing import sem_unlink
-        try: sem_unlink("non-existent")
-        except OSError: pass
+        try:
+            sem_unlink("non-existent")
+        except OSError:
+            pass
 
     def test_semaphore(self):
         from _multiprocessing import SemLock
@@ -35,7 +37,7 @@
         maxvalue = 1
         # the following line gets OSError: [Errno 38] Function not implemented
         # if /dev/shm is not mounted on Linux
-        sem = SemLock(kind, value, maxvalue, "1", False)
+        sem = SemLock(kind, value, maxvalue, "1", unlink=True)
         assert sem.kind == kind
         assert sem.maxvalue == maxvalue
         assert isinstance(sem.handle, int)
@@ -68,7 +70,7 @@
         maxvalue = 1
         # the following line gets OSError: [Errno 38] Function not implemented
         # if /dev/shm is not mounted on Linux
-        sem = SemLock(kind, value, maxvalue, "2", False)
+        sem = SemLock(kind, value, maxvalue, "2", unlink=True)
 
         sem.acquire()
         sem.release()
@@ -88,7 +90,7 @@
         kind = self.SEMAPHORE
         value = 1
         maxvalue = 1
-        sem = SemLock(kind, value, maxvalue, "3", False)
+        sem = SemLock(kind, value, maxvalue, "3", unlink=True)
 
         res = sem.acquire()
         assert res == True
@@ -100,7 +102,7 @@
         kind = self.SEMAPHORE
         value = 1
         maxvalue = 1
-        sem = SemLock(kind, value, maxvalue, "4", False)
+        sem = SemLock(kind, value, maxvalue, "4", unlink=True)
 
         sem2 = SemLock._rebuild(sem.handle, kind, value, "10")
         assert sem.handle == sem2.handle
@@ -110,8 +112,13 @@
         kind = self.SEMAPHORE
         value = 1
         maxvalue = 1
-        sem = SemLock(kind, value, maxvalue, "5", False)
+        sem = SemLock(kind, value, maxvalue, "5", unlink=True)
 
         with sem:
             assert sem._count() == 1
         assert sem._count() == 0
+
+    def test_unlink(self):
+        from _multiprocessing import SemLock
+        sem = SemLock(self.SEMAPHORE, 1, 1, '/mp-123', unlink=True)
+        assert sem._count() == 0
diff --git a/pypy/module/cpyext/test/test_ztranslation.py b/pypy/module/cpyext/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/cpyext/test/test_ztranslation.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-from pypy.module.cpyext import pyobject
-
-def test_cpyext_translates(monkeypatch):
-    def from_ref(space, ref):
-        # XXX: avoid 'assert isinstance(w_type, W_TypeObject)' from the
-        # original from_ref, just return w_some_obj
-        return space.w_object
-    monkeypatch.setattr(pyobject, 'from_ref', from_ref)
-    checkmodule('cpyext', '_rawffi', translate_startup=False)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -273,7 +273,7 @@
         if len(slot_names) == 1:
             if not getattr(pto, slot_names[0]):
                 setattr(pto, slot_names[0], slot_func_helper)
-        elif (w_type.getname(space) in ('list', 'tuple') and
+        elif ((w_type is space.w_list or w_type is space.w_tuple) and
               slot_names[0] == 'c_tp_as_number'):
             # XXX hack - hwo can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py
--- a/pypy/module/math/__init__.py
+++ b/pypy/module/math/__init__.py
@@ -11,6 +11,8 @@
     interpleveldefs = {
        'e'              : 'interp_math.get(space).w_e',
        'pi'             : 'interp_math.get(space).w_pi',
+       'inf'            : 'interp_math.get(space).w_inf',
+       'nan'            : 'interp_math.get(space).w_nan',
        'pow'            : 'interp_math.pow',
        'cosh'           : 'interp_math.cosh',
        'copysign'       : 'interp_math.copysign',
diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -1,4 +1,5 @@
 import sys
+from _operator import index
 
 def factorial(x):
     """factorial(x) -> Integral
@@ -45,8 +46,8 @@
 
 def gcd(x, y):
     """greatest common divisor of x and y"""
-    x = abs(x)
-    y = abs(y)
+    x = abs(index(x))
+    y = abs(index(y))
     while x > 0:
         x, y = y % x, x
     return y
diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -10,6 +10,8 @@
     def __init__(self, space):
         self.w_e = space.wrap(math.e)
         self.w_pi = space.wrap(math.pi)
+        self.w_inf = space.wrap(rfloat.INFINITY)
+        self.w_nan = space.wrap(rfloat.NAN)
 def get(space):
     return space.fromcache(State)
 
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -369,3 +369,10 @@
         assert math.gcd(-4, -10) == 2
         assert math.gcd(0, -10) == 10
         assert math.gcd(0, 0) == 0
+        raises(TypeError, math.gcd, 0, 0.0)
+
+    def test_inf_nan(self):
+        import math
+        assert math.isinf(math.inf)
+        assert math.inf > -math.inf
+        assert math.isnan(math.nan)
diff --git a/pypy/module/micronumpy/test/test_ztranslation.py b/pypy/module/micronumpy/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/micronumpy/test/test_ztranslation.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-
-def test_numpy_translates():
-    checkmodule('micronumpy')
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -56,6 +56,17 @@
             for attrs in self._multi_attrs
         ])
 
+    def __repr__(self):
+        try:
+            a = repr(self._simple_attr)
+        except AttributeError:
+            try:
+                a = repr('.'.join(self._single_attr))
+            except AttributeError:
+                lst = self._multi_attrs
+                a = ', '.join([repr('.'.join(a1)) for a1 in lst])
+        return 'operator.attrgetter(%s)' % (a,)
+
 
 class itemgetter(object):
     def __init__(self, item, *items):
@@ -71,12 +82,29 @@
         else:
             return tuple([obj[i] for i in self._idx])
 
+    def __repr__(self):
+        if self._single:
+            a = repr(self._idx)
+        else:
+            a = ', '.join([repr(i) for i in self._idx])
+        return 'operator.itemgetter(%s)' % (a,)
+
 
 class methodcaller(object):
     def __init__(self, method_name, *args, **kwargs):
+        if not isinstance(method_name, str):
+            raise TypeError("method name must be a string")
         self._method_name = method_name
         self._args = args
         self._kwargs = kwargs
 
     def __call__(self, obj):
         return getattr(obj, self._method_name)(*self._args, **self._kwargs)
+
+    def __repr__(self):
+        args = [repr(self._method_name)]
+        for a in self._args:
+            args.append(repr(a))
+        for key, value in self._kwargs.items():
+            args.append('%s=%r' % (key, value))
+        return 'operator.methodcaller(%s)' % (', '.join(args),)
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -210,7 +210,7 @@
     return space.inplace_add(w_obj1, w_obj2)
 
 @unwrap_spec(default=int)
-def length_hint(space, w_iterable, default):
+def length_hint(space, w_iterable, default=0):
     """Return an estimate of the number of items in obj.
     This is useful for presizing containers when building from an iterable.
     If the object supports len(), the result will be exact.
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -182,6 +182,11 @@
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
+    def test_methodcaller_not_string(self):
+        import _operator as operator
+        e = raises(TypeError, operator.methodcaller, 42)
+        assert str(e.value) == "method name must be a string"
+
     def test_index(self):
         import _operator as operator
         assert operator.index(42) == 42
@@ -322,3 +327,27 @@
         assert operator._compare_digest(u'asd', u'asd')
         assert not operator._compare_digest(u'asd', u'qwe')
         raises(TypeError, operator._compare_digest, u'asd', b'qwe')
+
+    def test_length_hint(self):
+        import _operator as operator
+        assert operator.length_hint([1, 2]) == 2
+
+    def test_repr_attrgetter(self):
+        import _operator as operator
+        assert repr(operator.attrgetter("foo")) == "operator.attrgetter('foo')"
+        assert repr(operator.attrgetter("foo", 'bar')) == (
+            "operator.attrgetter('foo', 'bar')")
+        assert repr(operator.attrgetter("foo.bar")) == (
+            "operator.attrgetter('foo.bar')")
+        assert repr(operator.attrgetter("foo", 'bar.baz')) == (
+            "operator.attrgetter('foo', 'bar.baz')")
+
+    def test_repr_itemgetter(self):
+        import _operator as operator
+        assert repr(operator.itemgetter(2)) == "operator.itemgetter(2)"
+        assert repr(operator.itemgetter(2, 3)) == "operator.itemgetter(2, 3)"
+
+    def test_repr_methodcaller(self):
+        import _operator as operator
+        assert repr(operator.methodcaller("foo", "bar", baz=42)) == (
+            "operator.methodcaller('foo', 'bar', baz=42)")
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -108,6 +108,27 @@
     return _unpack(space, format, buf)
 
 
+class W_UnpackIter(W_Root):
+    def __init__(self, w_struct, buf):
+        self.w_struct = w_struct
+        self.buf = buf
+        self.index = 0
+
+    def descr_iter(self, space):
+        return self
+
+    def descr_next(self, space):
+        if self.w_struct is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.index >= self.buf.getlength():
+            raise OperationError(space.w_StopIteration, space.w_None)
+        size = self.w_struct.size
+        buf = SubBuffer(self.buf, self.index, size)
+        w_res = _unpack(space, self.w_struct.format, buf)
+        self.index += size
+        return w_res
+
+
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
@@ -135,6 +156,10 @@
     def descr_unpack_from(self, space, w_buffer, offset=0):
         return unpack_from(space, jit.promote_string(self.format), w_buffer, offset)
 
+    def descr_iter_unpack(self, space, w_buffer):
+        buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+        return W_UnpackIter(self, buf)
+
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
     format=interp_attrproperty_bytes("format", cls=W_Struct),
@@ -144,6 +169,21 @@
     unpack=interp2app(W_Struct.descr_unpack),
     pack_into=interp2app(W_Struct.descr_pack_into),
     unpack_from=interp2app(W_Struct.descr_unpack_from),
+    iter_unpack=interp2app(W_Struct.descr_iter_unpack),
+)
+
+ at unwrap_spec(w_struct=W_Struct)
+def new_unpackiter(space, w_subtype, w_struct, w_buffer):
+    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+    w_res = space.allocate_instance(W_UnpackIter, w_subtype)
+    w_res.__init__(w_struct, buf)
+    return w_res
+
+W_UnpackIter.typedef = TypeDef("unpack_iterator",
+    __new__=interp2app(new_unpackiter),
+    __iter__=interp2app(W_UnpackIter.descr_iter),
+    __next__=interp2app(W_UnpackIter.descr_next),
+    #__length_hint__=
 )
 
 def clearcache(space):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -394,6 +394,13 @@
         exc = raises(self.struct.error, self.struct.unpack_from, "ii", memoryview(b''))
         assert str(exc.value) == "unpack_from requires a buffer of at least 8 bytes"
 
+    def test_iter_unpack(self):
+        import array
+        b = array.array('b', b'\0' * 16)
+        s = self.struct.Struct('ii')
+        it = s.iter_unpack(b)
+        assert list(it) == [(0, 0), (0, 0)]
+
     def test___float__(self):
         class MyFloat(object):
             def __init__(self, x):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1356,15 +1356,15 @@
         assert ffi.getctype("e1*") == 'e1 *'
 
     def test_opaque_enum(self):
+        import warnings
         ffi = FFI(backend=self.Backend())
         ffi.cdef("enum foo;")
-        from cffi import __version_info__
-        if __version_info__ < (1, 8):
-            py.test.skip("re-enable me in version 1.8")
-        e = py.test.raises(CDefError, ffi.cast, "enum foo", -1)
-        assert str(e.value) == (
-            "'enum foo' has no values explicitly defined: refusing to guess "
-            "which integer type it is meant to be (unsigned/signed, int/long)")
+        with warnings.catch_warnings(record=True) as log:
+            n = ffi.cast("enum foo", -1)
+            assert int(n) == 0xffffffff
+        assert str(log[0].message) == (
+            "'enum foo' has no values explicitly defined; "
+            "guessing that it is equivalent to 'unsigned int'")
 
     def test_new_ctype(self):
         ffi = FFI(backend=self.Backend())
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -283,10 +283,20 @@
         ffi.cdef("struct foo_s { int x; int a[]; };")
         p = ffi.new("struct foo_s *", [100, [200, 300, 400]])
         assert p.x == 100
-        assert ffi.typeof(p.a) is ffi.typeof("int *")   # no length available
+        assert ffi.typeof(p.a) is ffi.typeof("int[]")
+        assert len(p.a) == 3                            # length recorded
         assert p.a[0] == 200
         assert p.a[1] == 300
         assert p.a[2] == 400
+        assert list(p.a) == [200, 300, 400]
+        q = ffi.cast("struct foo_s *", p)
+        assert q.x == 100
+        assert ffi.typeof(q.a) is ffi.typeof("int *")   # no length recorded
+        py.test.raises(TypeError, len, q.a)
+        assert q.a[0] == 200
+        assert q.a[1] == 300
+        assert q.a[2] == 400
+        py.test.raises(TypeError, list, q.a)
 
     @pytest.mark.skipif("sys.platform != 'win32'")
     def test_getwinerror(self):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -562,7 +562,8 @@
                      "int bar(struct foo_s *f) { return f->a[14]; }\n")
     assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *")
-    assert ffi.typeof(s.a) is ffi.typeof('int *')   # because no length
+    assert ffi.typeof(s.a) is ffi.typeof('int[]')   # implicit max length
+    assert len(s.a) == 18  # max length, computed from the size and start offset
     s.a[14] = 4242
     assert lib.bar(s) == 4242
     # with no declared length, out-of-bound accesses are not detected
@@ -592,10 +593,15 @@
     ffi.verify("struct foo_s { int x; int a[]; };")
     assert ffi.sizeof('struct foo_s') == 1 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *", [424242, 4])
-    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')   # the same in C
+    assert ffi.sizeof(ffi.typeof(s[0])) == 1 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 5 * ffi.sizeof('int')
+    # ^^^ explanation: if you write in C: "char x[5];", then
+    # "sizeof(x)" will evaluate to 5.  The behavior above is
+    # a generalization of that to "struct foo_s[len(a)=5] x;"
+    # if you could do that in C.
     assert s.a[3] == 0
     s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
-    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 5 * ffi.sizeof('int')
     assert s.a[3] == -10
     s = ffi.new("struct foo_s *")
     assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
@@ -610,10 +616,10 @@
     ffi.verify("struct foo_s { int x, y; int a[]; };")
     assert ffi.sizeof('struct foo_s') == 2 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *", [424242, 4])
-    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 6 * ffi.sizeof('int')
     assert s.a[3] == 0
     s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
-    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 6 * ffi.sizeof('int')
     assert s.a[3] == -10
     s = ffi.new("struct foo_s *")
     assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1634,10 +1634,19 @@
         # struct array_no_length { int x; int a[]; };
         p = ffi.new("struct array_no_length *", [100, [200, 300, 400]])
         assert p.x == 100
-        assert ffi.typeof(p.a) is ffi.typeof("int *")   # no length available
+        assert ffi.typeof(p.a) is ffi.typeof("int[]")   # length available
         assert p.a[0] == 200
         assert p.a[1] == 300
         assert p.a[2] == 400
+        assert len(p.a) == 3
+        assert list(p.a) == [200, 300, 400]
+        q = ffi.cast("struct array_no_length *", p)
+        assert ffi.typeof(q.a) is ffi.typeof("int *")   # no length available
+        assert q.a[0] == 200
+        assert q.a[1] == 300
+        assert q.a[2] == 400
+        py.test.raises(TypeError, len, q.a)
+        py.test.raises(TypeError, list, q.a)
 
     def test_from_buffer(self):
         import array
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -400,11 +400,14 @@
         pass    # ok, fail during compilation already (e.g. C++)
     else:
         assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
-        p = ffi.new("struct foo_s *")
-        # lazily build the fields and boom:
-        e = py.test.raises(ffi.error, "p.a")
-        assert str(e.value).startswith("struct foo_s: wrong size for field 'a' "
-                                       "(cdef says 20, but C compiler says 24)")
+        try:
+            # lazily build the fields and boom:
+            p = ffi.new("struct foo_s *")
+            p.a
+            assert False, "should have raised"
+        except ffi.error as e:
+            assert str(e).startswith("struct foo_s: wrong size for field 'a' "
+                                     "(cdef says 20, but C compiler says 24)")
 
 def test_open_array_in_struct():
     ffi = FFI()
@@ -412,8 +415,10 @@
     verify(ffi, 'test_open_array_in_struct',
            "struct foo_s { int b; int a[]; };")
     assert ffi.sizeof("struct foo_s") == 4
-    p = ffi.new("struct foo_s *", [5, [10, 20, 30]])
+    p = ffi.new("struct foo_s *", [5, [10, 20, 30, 40]])
     assert p.a[2] == 30
+    assert ffi.sizeof(p) == ffi.sizeof("void *")
+    assert ffi.sizeof(p[0]) == 5 * ffi.sizeof("int")
 
 def test_math_sin_type():
     ffi = FFI()
@@ -999,6 +1004,7 @@
                  "struct foo_s { int x; int a[5][8]; int y; };")
     assert ffi.sizeof('struct foo_s') == 42 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *")
+    assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
     assert ffi.sizeof(s.a) == 40 * ffi.sizeof('int')
     assert s.a[4][7] == 0
     py.test.raises(IndexError, 's.a[4][8]')
@@ -1013,7 +1019,7 @@
                  "struct foo_s { int x; int a[5][7]; int y; };")
     assert ffi.sizeof('struct foo_s') == 37 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *")
-    assert ffi.typeof(s.a) == ffi.typeof("int(*)[7]")
+    assert ffi.typeof(s.a) == ffi.typeof("int[][7]")
     assert s.a[4][6] == 0
     py.test.raises(IndexError, 's.a[4][7]')
     assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -547,7 +547,8 @@
                      "int bar(struct foo_s *f) { return f->a[14]; }\n")
     assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *")
-    assert ffi.typeof(s.a) is ffi.typeof('int *')   # because no length
+    assert ffi.typeof(s.a) is ffi.typeof('int[]')   # implicit max length
+    assert len(s.a) == 18  # max length, computed from the size and start offset
     s.a[14] = 4242
     assert lib.bar(s) == 4242
     # with no declared length, out-of-bound accesses are not detected
@@ -577,10 +578,15 @@
     ffi.verify("struct foo_s { int x; int a[]; };")
     assert ffi.sizeof('struct foo_s') == 1 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *", [424242, 4])
-    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')   # the same in C
+    assert ffi.sizeof(ffi.typeof(s[0])) == 1 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 5 * ffi.sizeof('int')
+    # ^^^ explanation: if you write in C: "char x[5];", then
+    # "sizeof(x)" will evaluate to 5.  The behavior above is
+    # a generalization of that to "struct foo_s[len(a)=5] x;"
+    # if you could do that in C.
     assert s.a[3] == 0
     s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
-    assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 5 * ffi.sizeof('int')
     assert s.a[3] == -10
     s = ffi.new("struct foo_s *")
     assert ffi.sizeof(s[0]) == 1 * ffi.sizeof('int')
@@ -595,10 +601,10 @@
     ffi.verify("struct foo_s { int x, y; int a[]; };")
     assert ffi.sizeof('struct foo_s') == 2 * ffi.sizeof('int')
     s = ffi.new("struct foo_s *", [424242, 4])
-    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 6 * ffi.sizeof('int')
     assert s.a[3] == 0
     s = ffi.new("struct foo_s *", [424242, [-40, -30, -20, -10]])
-    assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
+    assert ffi.sizeof(s[0]) == 6 * ffi.sizeof('int')
     assert s.a[3] == -10
     s = ffi.new("struct foo_s *")
     assert ffi.sizeof(s[0]) == 2 * ffi.sizeof('int')
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -291,6 +291,9 @@
     def type(self, w_obj):
         return w_some_type()
 
+    def lookup_in_type_where(self, w_type, key):
+        return w_some_obj(), w_some_obj()
+
     def issubtype_w(self, w_sub, w_type):
         is_root(w_sub)
         is_root(w_type)
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -132,12 +132,11 @@
     return format_float(x, 'g', DTSF_STR_PRECISION)
 
 
-def unpackcomplex(space, w_complex, strict_typing=True):
+def unpackcomplex(space, w_complex):
     """
-    convert w_complex into a complex and return the unwrapped (real, imag)
-    tuple. If strict_typing==True, we also typecheck the value returned by
-    __complex__ to actually be a complex (and not e.g. a float).
-    See test___complex___returning_non_complex.
+    Convert w_complex into a complex and return the unwrapped (real, imag)
+    tuple. Also, typecheck the value returned by __complex__ to actually be a
+    complex (and not e.g. a float).
     """
     if type(w_complex) is W_ComplexObject:
         return (w_complex.realval, w_complex.imagval)
@@ -149,12 +148,9 @@
         w_z = space.get_and_call_function(w_method, w_complex)
     #
     if w_z is not None:
-        # __complex__() must return a complex or (float,int,long) object
+        # __complex__() must return a complex
         # (XXX should not use isinstance here)
-        if not strict_typing and (space.isinstance_w(w_z, space.w_int) or
-                                  space.isinstance_w(w_z, space.w_float)):
-            return (space.float_w(w_z), 0.0)
-        elif isinstance(w_z, W_ComplexObject):
+        if isinstance(w_z, W_ComplexObject):
             return (w_z.realval, w_z.imagval)
         raise oefmt(space.w_TypeError,
                     "__complex__() must return a complex number")
@@ -302,14 +298,12 @@
 
         else:
             # non-string arguments
-            realval, imagval = unpackcomplex(space, w_real,
-                                             strict_typing=False)
+            realval, imagval = unpackcomplex(space, w_real)
 
             # now take w_imag into account
             if not noarg2:
                 # complex(x, y) == x+y*j, even if 'y' is already a complex.
-                realval2, imagval2 = unpackcomplex(space, w_imag,
-                                                   strict_typing=False)
+                realval2, imagval2 = unpackcomplex(space, w_imag)
 
                 # try to preserve the signs of zeroes of realval and realval2
                 if imagval2 != 0.0:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -612,14 +612,8 @@
 
     def _startswith(self, space, value, w_prefix, start, end):
         prefix = self._op_val(space, w_prefix)
-        if start > len(value):
-            return self._starts_ends_overflow(prefix)
         return startswith(value, prefix, start, end)
 
-    def _starts_ends_overflow(self, prefix):
-        return False     # bug-to-bug compat: this is for strings and
-                         # bytearrays, but overridden for unicodes
-
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_suffix, space.w_tuple):
@@ -643,8 +637,6 @@
 
     def _endswith(self, space, value, w_prefix, start, end):
         prefix = self._op_val(space, w_prefix)
-        if start > len(value):
-            return self._starts_ends_overflow(prefix)
         return endswith(value, prefix, start, end)
 
     def _strip(self, space, w_chars, left, right):
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -265,41 +265,26 @@
         assert self.close(complex(5.3, 9.8).conjugate(), 5.3-9.8j)
 
     def test_constructor(self):
-        class OS:
-            def __init__(self, value):
-                self.value = value
-            def __complex__(self):
-                return self.value
         class NS(object):
             def __init__(self, value):
                 self.value = value
             def __complex__(self):
                 return self.value
-        assert complex(OS(1+10j)) == 1+10j
         assert complex(NS(1+10j)) == 1+10j
-        assert complex(OS(1+10j), 5) == 1+15j
         assert complex(NS(1+10j), 5) == 1+15j
-        assert complex(OS(1+10j), 5j) == -4+10j
         assert complex(NS(1+10j), 5j) == -4+10j
-        assert complex(OS(2.0)) == 2+0j
-        assert complex(NS(2.0)) == 2+0j
-        assert complex(OS(2)) == 2+0j
-        assert complex(NS(2)) == 2+0j
-        raises(TypeError, complex, OS(None))
+        raises(TypeError, complex, NS(2.0))
+        raises(TypeError, complex, NS(2))
         raises(TypeError, complex, NS(None))
         raises(TypeError, complex, b'10')
 
         # -- The following cases are not supported by CPython, but they
         # -- are supported by PyPy, which is most probably ok
-        #raises((TypeError, AttributeError), complex, OS(1+10j), OS(1+10j))
-        #raises((TypeError, AttributeError), complex, NS(1+10j), OS(1+10j))
-        #raises((TypeError, AttributeError), complex, OS(1+10j), NS(1+10j))
         #raises((TypeError, AttributeError), complex, NS(1+10j), NS(1+10j))
 
         class F(object):
             def __float__(self):
                 return 2.0
-        assert complex(OS(1+10j), F()) == 1+12j
         assert complex(NS(1+10j), F()) == 1+12j
 
         assert self.almost_equal(complex("1+10j"), 1+10j)
@@ -393,23 +378,6 @@
         assert complex(s) == 1+2j
         assert complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) ') == 1+1j
 
-    def test___complex___returning_non_complex(self):
-        import cmath
-        class Obj(object):
-            def __init__(self, value):
-                self.value = value
-            def __complex__(self):
-                return self.value
-
-        # "bug-to-bug" compatibility to CPython: complex() is more relaxed in
-        # what __complex__ can return. cmath functions really wants a complex
-        # number to be returned by __complex__.
-        assert complex(Obj(2.0)) == 2+0j
-        assert complex(Obj(2)) == 2+0j
-        #
-        assert cmath.polar(1) == (1.0, 0.0)
-        raises(TypeError, "cmath.polar(Obj(1))")
-
     def test_hash(self):
         for x in range(-30, 30):
             assert hash(x) == hash(complex(x, 0))

From pypy.commits at gmail.com  Tue Nov  1 09:44:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 06:44:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58189c40.28c9c20a.d5941.3f89@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88031:ec98ae5d2272
Date: 2016-11-01 14:43 +0100
http://bitbucket.org/pypy/pypy/changeset/ec98ae5d2272/

Log:	hg merge default

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):

From pypy.commits at gmail.com  Tue Nov  1 11:20:44 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 01 Nov 2016 08:20:44 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: clear the forwarding
 information of the prefixed operations as well
Message-ID: <5818b2cc.2636c20a.2808.15f8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88032:ba7a27ee1304
Date: 2016-11-01 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/ba7a27ee1304/

Log:	clear the forwarding information of the prefixed operations as well

diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -83,7 +83,12 @@
             oplist = self.prefix
         if label:
             oplist = [self.label] + oplist
-        return oplist + self.operations + [self.jump]
+        if label != True:
+            for op in oplist:
+                op.set_forwarded(None)
+            self.jump.set_forwarded(None)
+        ops = oplist + self.operations + [self.jump]
+        return ops
 
     def clone(self):
         renamer = Renamer()

From pypy.commits at gmail.com  Tue Nov  1 11:29:50 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 01 Nov 2016 08:29:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Try to fix translation on linux32
Message-ID: <5818b4ee.c1d81c0a.b9f88.b768@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88033:c715b57ada9b
Date: 2016-11-01 16:28 +0100
http://bitbucket.org/pypy/pypy/changeset/c715b57ada9b/

Log:	Try to fix translation on linux32

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -102,6 +102,8 @@
     INITIAL_BUFFER_SIZE = 8192
 
 UINT_MAX = 2**32-1
+MAX_BUFSIZE = min(sys.maxint, UINT_MAX)
+assert isinstance(MAX_BUFSIZE, int)
 
 if rffi.sizeof(rffi.INT) > 4:
     BIGCHUNK = 512 * 32
@@ -455,7 +457,7 @@
 
     def _decompress_buf(self, data, max_length):
         total_in = len(data)
-        in_bufsize = min(total_in, UINT_MAX)
+        in_bufsize = min(total_in, MAX_BUFSIZE)
         total_in -= in_bufsize
         with rffi.scoped_nonmovingbuffer(data) as in_buf:
             # setup the input and the size it can consume

From pypy.commits at gmail.com  Tue Nov  1 11:41:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Nov 2016 08:41:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <5818b7c0.e8edc20a.9a8b.6a71@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88034:6dd9cf87c0a7
Date: 2016-11-01 15:41 +0000
http://bitbucket.org/pypy/pypy/changeset/6dd9cf87c0a7/

Log:	fix translation

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -578,7 +578,7 @@
         self.pop_frame_block(F_BLOCK_LOOP, start)
         self.visit_sequence(fr.orelse)
         self.use_next_block(end)
-    
+
     def visit_AsyncFor(self, fr):
         self.update_position(fr.lineno, True)
         b_try = self.new_block()
@@ -588,21 +588,21 @@
         b_try_cleanup = self.new_block()
         b_after_loop = self.new_block()
         b_after_loop_else = self.new_block()
-        
+
         self.emit_jump(ops.SETUP_LOOP, b_after_loop)
         self.push_frame_block(F_BLOCK_LOOP, b_try)
-        
+
         fr.iter.walkabout(self)
         self.emit_op(ops.GET_AITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
-        
+
         self.use_next_block(b_try)
         # This adds another line, so each for iteration can be traced.
         self.lineno_set = False
         self.emit_jump(ops.SETUP_EXCEPT, b_except)
         self.push_frame_block(F_BLOCK_EXCEPT, b_try)
-        
+
         self.emit_op(ops.GET_ANEXT)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
@@ -610,13 +610,13 @@
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_EXCEPT, b_try)
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
-        
+
         self.use_next_block(b_except)
         self.emit_op(ops.DUP_TOP)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)
-        
+
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
@@ -630,23 +630,23 @@
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.emit_jump(ops.JUMP_ABSOLUTE, b_after_loop_else, True)
-        
+
         self.use_next_block(b_try_cleanup)
         self.emit_op(ops.END_FINALLY)
-        
+
         self.use_next_block(b_after_try)
         self.visit_sequence(fr.body)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True)
-        
+
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.pop_frame_block(F_BLOCK_LOOP, b_try)
-        
+
         self.use_next_block(b_after_loop)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True)
-        
+
         self.use_next_block(b_after_loop_else)
         self.visit_sequence(fr.orelse)
-        
+
         self.use_next_block(b_end)
 
     def visit_While(self, wh):
@@ -701,7 +701,7 @@
             self.emit_op(ops.POP_TOP)
             if handler.name:
                 ## generate the equivalent of:
-                ## 
+                ##
                 ## try:
                 ##     # body
                 ## except type as name:
@@ -960,7 +960,7 @@
 
     def visit_AsyncWith(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_asyncwithitem(wih, 0, is_async=True)
+        self.handle_withitem(wih, 0, is_async=True)
 
     def visit_Raise(self, rais):
         self.update_position(rais.lineno, True)
@@ -1008,7 +1008,7 @@
         self.emit_op(ops.GET_YIELD_FROM_ITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
-    
+
     def visit_Await(self, aw):
         self.update_position(aw.lineno)
         aw.value.walkabout(self)
@@ -1147,7 +1147,7 @@
             self.emit_op_arg(outer_op, elt_subitems)
         else:
             self.emit_op_arg(single_op, seen_star)
-    
+
     def _visit_assignment(self, node, elts, ctx):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
@@ -1230,7 +1230,7 @@
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
             is_unpacking = False
-    
+
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
 
@@ -1286,7 +1286,7 @@
                     # If we ended up with more than one stararg, we need
                     # to concatenate them into a single sequence.
                     self.emit_op_arg(ops.BUILD_LIST_UNPACK, nsubargs)
-        
+
         # Repeat procedure for keyword args
         nseen = 0 # the number of keyword arguments on the stack following
         if keywords is not None:
@@ -1340,7 +1340,7 @@
             return
         call.func.walkabout(self)
         self._make_call(0, call.args, call.keywords)
-    
+
     def _call_has_no_star_args(self, call):
         if call.args is not None:
             for elt in call.args:

From pypy.commits at gmail.com  Tue Nov  1 11:50:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Nov 2016 08:50:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement struct.iter_unpack() function
Message-ID: <5818b9da.04831c0a.f55f1.b912@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88035:bbc8635cc762
Date: 2016-11-01 15:50 +0000
http://bitbucket.org/pypy/pypy/changeset/bbc8635cc762/

Log:	Implement struct.iter_unpack() function

diff --git a/pypy/module/struct/__init__.py b/pypy/module/struct/__init__.py
--- a/pypy/module/struct/__init__.py
+++ b/pypy/module/struct/__init__.py
@@ -55,6 +55,7 @@
         'pack_into': 'interp_struct.pack_into',
         'unpack': 'interp_struct.unpack',
         'unpack_from': 'interp_struct.unpack_from',
+        'iter_unpack': 'interp_struct.iter_unpack',
 
         'Struct': 'interp_struct.W_Struct',
         '_clearcache': 'interp_struct.clearcache',
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -186,5 +186,11 @@
     #__length_hint__=
 )
 
+ at unwrap_spec(format=str)
+def iter_unpack(space, format, w_buffer):
+    w_struct = W_Struct(space, format)
+    buf = space.buffer_w(w_buffer, space.BUF_SIMPLE)
+    return W_UnpackIter(w_struct, buf)
+
 def clearcache(space):
     """No-op on PyPy"""
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -400,6 +400,8 @@
         s = self.struct.Struct('ii')
         it = s.iter_unpack(b)
         assert list(it) == [(0, 0), (0, 0)]
+        it = self.struct.iter_unpack('ii', b)
+        assert list(it) == [(0, 0), (0, 0)]
 
     def test___float__(self):
         class MyFloat(object):

From pypy.commits at gmail.com  Tue Nov  1 12:03:17 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Nov 2016 09:03:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add unpack_iterator.__length_hint__()
Message-ID: <5818bcc5.87b2c20a.fa67.67ee@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88036:8596145cda3f
Date: 2016-11-01 16:02 +0000
http://bitbucket.org/pypy/pypy/changeset/8596145cda3f/

Log:	Add unpack_iterator.__length_hint__()

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -128,6 +128,12 @@
         self.index += size
         return w_res
 
+    def descr_length_hint(self, space):
+        if self.w_struct is None:
+            return space.newint(0)
+        length = (self.buf.getlength() - self.index) // self.w_struct.size
+        return space.newint(length)
+
 
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
@@ -183,7 +189,7 @@
     __new__=interp2app(new_unpackiter),
     __iter__=interp2app(W_UnpackIter.descr_iter),
     __next__=interp2app(W_UnpackIter.descr_next),
-    #__length_hint__=
+    __length_hint__=interp2app(W_UnpackIter.descr_length_hint)
 )
 
 @unwrap_spec(format=str)
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -399,6 +399,7 @@
         b = array.array('b', b'\0' * 16)
         s = self.struct.Struct('ii')
         it = s.iter_unpack(b)
+        assert it.__length_hint__() == 2
         assert list(it) == [(0, 0), (0, 0)]
         it = self.struct.iter_unpack('ii', b)
         assert list(it) == [(0, 0), (0, 0)]

From pypy.commits at gmail.com  Tue Nov  1 12:31:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 09:31:14 -0700 (PDT)
Subject: [pypy-commit] pypy openssl-1.1: Fix for Windows: fails in
 untranslated tests if the TLS_method()
Message-ID: <5818c352.04831c0a.f55f1.c899@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: openssl-1.1
Changeset: r88037:3f3f9319d227
Date: 2016-11-01 17:30 +0100
http://bitbucket.org/pypy/pypy/changeset/3f3f9319d227/

Log:	Fix for Windows: fails in untranslated tests if the TLS_method()
	function is made 'macro=True'. Not sure I want to dig into the
	reason for that mess.

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -62,9 +62,6 @@
         '#if (OPENSSL_VERSION_NUMBER < 0x10100000)\n'
         '#  define COMP_get_name(meth) (meth->name)\n'
         '#  define COMP_get_type(meth) (meth->type)\n'
-        '#  define EVP_MD_CTX_free EVP_MD_CTX_destroy\n'
-        '#  define EVP_MD_CTX_new EVP_MD_CTX_create\n'
-        '#  define TLS_method SSLv23_method\n'
         '#  define X509_NAME_ENTRY_set(ne) (ne->set)\n'
         '#  define X509_OBJECT_get0_X509(obj) (obj->data.x509)\n'
         '#  define X509_OBJECT_get_type(obj) (obj->type)\n'
@@ -314,8 +311,12 @@
     ssl_external('TLSv1_2_method', [], SSL_METHOD)
 ssl_external('SSLv2_method', [], SSL_METHOD)
 ssl_external('SSLv3_method', [], SSL_METHOD)
-ssl_external('TLS_method', [], SSL_METHOD,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+# Windows note: fails in untranslated tests if the following function is
+# made 'macro=True'.  Not sure I want to dig into the reason for that mess.
+libssl_TLS_method = external(
+    'TLS_method' if OPENSSL_VERSION_NUMBER >= 0x10100000
+    else 'SSLv23_method',
+    [], SSL_METHOD)
 ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT,
@@ -498,7 +499,10 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
+libssl_OPENSSL_free = external(
+    'OPENSSL_free' if OPENSSL_VERSION_NUMBER >= 0x10100000
+    else 'CRYPTO_free',
+    [rffi.VOIDP], lltype.Void)
 
 ssl_external('SSL_write', [SSL, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=SAVE_ERR)
@@ -586,11 +590,13 @@
 EVP_MD_CTX_copy = external(
     'EVP_MD_CTX_copy', [EVP_MD_CTX, EVP_MD_CTX], rffi.INT)
 EVP_MD_CTX_new = external(
-    'EVP_MD_CTX_new', [], EVP_MD_CTX,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+    'EVP_MD_CTX_new' if OPENSSL_VERSION_NUMBER >= 0x10100000
+    else 'EVP_MD_CTX_create',
+    [], EVP_MD_CTX)
 EVP_MD_CTX_free = external(
-    'EVP_MD_CTX_free', [EVP_MD_CTX], lltype.Void, releasegil=False,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+    'EVP_MD_CTX_free' if OPENSSL_VERSION_NUMBER >= 0x10100000
+    else 'EVP_MD_CTX_destroy',
+    [EVP_MD_CTX], lltype.Void, releasegil=False)
 
 if OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL:
     PKCS5_PBKDF2_HMAC = external('PKCS5_PBKDF2_HMAC', [

From pypy.commits at gmail.com  Tue Nov  1 12:58:11 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 01 Nov 2016 09:58:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement the codec "namereplace" error
 handler.
Message-ID: <5818c9a3.88abc20a.f34a6.8713@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88038:d581f95a1c58
Date: 2016-11-01 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/d581f95a1c58/

Log:	Implement the codec "namereplace" error handler. u'\uabcd' ==>
	b'\N{SOME NAME}'

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,10 +1,11 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
 from rpython.rlib.runicode import code_to_unichr, MAXUNICODE
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.module.unicodedata import unicodedb
 
 
 class VersionTag(object):
@@ -295,6 +296,26 @@
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
 
+def namereplace_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        builder = StringBuilder()
+        pos = start
+        while pos < end:
+            oc = ord(obj[pos])
+            builder.append('\\N{')
+            builder.append(unicodedb.name(oc))
+            builder.append('}')
+            pos += 1
+        return space.newtuple([space.newbytes(builder.build()), w_end])
+    else:
+        raise oefmt(space.w_TypeError,
+                    "don't know how to handle %T in error callback", w_exc)
+
 def surrogatepass_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
@@ -382,7 +403,8 @@
     "NOT_RPYTHON"
     state = space.fromcache(CodecState)
     for error in ("strict", "ignore", "replace", "xmlcharrefreplace",
-                  "backslashreplace", "surrogateescape", "surrogatepass"):
+                  "backslashreplace", "surrogateescape", "surrogatepass",
+                  "namereplace"):
         name = error + "_errors"
         state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -561,9 +561,14 @@
         assert b'\xff'.decode('utf-7', 'ignore') == ''
         assert b'\x00'.decode('unicode-internal', 'ignore') == ''
 
-    def test_backslahreplace(self):
+    def test_backslashreplace(self):
         assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000'
 
+    def test_namereplace(self):
+        assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'namereplace') == (
+            b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}'
+            b'\\N{CJK UNIFIED IDEOGRAPH-8000}')
+
     def test_surrogateescape(self):
         assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b'
         assert 'a\udc80b'.encode('utf-8', 'surrogateescape') == b'a\x80b'

From pypy.commits at gmail.com  Tue Nov  1 12:58:14 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 01 Nov 2016 09:58:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix error messages to match CPython 3.5
Message-ID: <5818c9a6.c24bc20a.123fd.847e@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88040:879102e55ef2
Date: 2016-11-01 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/879102e55ef2/

Log:	Fix error messages to match CPython 3.5

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -689,7 +689,7 @@
             return errorchar
 
         raise oefmt(space.w_TypeError,
-            "character mapping must return integer, None or unicode")
+            "character mapping must return integer, None or str")
 
 class Charmap_Encode:
     def __init__(self, space, w_mapping):
@@ -722,7 +722,7 @@
             return errorchar
 
         raise oefmt(space.w_TypeError,
-            "character mapping must return integer, None or str")
+            "character mapping must return integer, bytes or None, not str")
 
 
 @unwrap_spec(string='bufferstr', errors='str_or_None')
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -112,7 +112,7 @@
         assert charmap_decode(b'xxx\xff', 'strict', map) == ('xxx\xff', 4)
 
         exc = raises(TypeError, charmap_decode, b'\xff', "strict",  {0xff: b'a'})
-        assert str(exc.value) == "character mapping must return integer, None or unicode"
+        assert str(exc.value) == "character mapping must return integer, None or str"
         raises(TypeError, charmap_decode, b'\xff', "strict",  {0xff: 0x110000})
         assert (charmap_decode(b"\x00\x01\x02", "strict",
                                {0: 0x10FFFF, 1: ord('b'), 2: ord('c')}) ==
@@ -687,7 +687,7 @@
         exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: 300})
         assert str(exc.value) == 'character mapping must be in range(256)'
         exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: u'a'})
-        assert str(exc.value) == 'character mapping must return integer, None or str'
+        assert str(exc.value) == 'character mapping must return integer, bytes or None, not str'
         raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None})
 
     def test_charmap_encode_replace(self):

From pypy.commits at gmail.com  Tue Nov  1 12:58:13 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 01 Nov 2016 09:58:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test. Testing with -A yields the same
 error.
Message-ID: <5818c9a5.4b1d1c0a.d3cb0.d539@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88039:82de3e6824df
Date: 2016-11-01 17:50 +0100
http://bitbucket.org/pypy/pypy/changeset/82de3e6824df/

Log:	Fix test. Testing with -A yields the same error.

diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -50,7 +50,7 @@
                     ]
         for s in insecure:
             buf = b"S" + s + b"\012p0\012."
-            raises (ValueError, pickle.loads, buf)
+            raises ((ValueError, pickle.UnpicklingError), pickle.loads, buf)
 
     def test_unicodedecodeerror(self):
         assert str(UnicodeDecodeError(

From pypy.commits at gmail.com  Tue Nov  1 13:11:45 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 10:11:45 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: remove some str_w calls
Message-ID: <5818ccd1.6c8ec20a.eca8f.9388@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88041:5fef2e96ef36
Date: 2016-11-01 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/5fef2e96ef36/

Log:	remove some str_w calls

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -359,7 +359,7 @@
     i = 0
     for w_key in keys_w:
         try:
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError, "keywords must be strings")
@@ -538,7 +538,7 @@
                             w_err = space.newtext("replace")
                             w_name = space.call_method(w_name, "encode", w_enc,
                                                        w_err)
-                            name = space.str_w(w_name)
+                            name = space.text_w(w_name)
                     break
         self.kwd_name = name
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -636,7 +636,7 @@
         w_keys = self.call_method(w_dic, "keys")
         exc_types_w = {}
         for w_name in self.unpackiterable(w_keys):
-            name = self.str_w(w_name)
+            name = self.text_w(w_name)
             if not name.startswith('__'):
                 excname = name
                 w_exc = self.getitem(w_dic, w_name)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -26,7 +26,7 @@
 # helper
 
 def unpack_str_tuple(space,w_str_tuple):
-    return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
+    return [space.text_w(w_el) for w_el in space.unpackiterable(w_str_tuple)]
 
 
 # Magic numbers for the bytecode version in code objects.
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -468,7 +468,7 @@
         return self.getcode().co_consts_w[index]
 
     def getname_u(self, index):
-        return self.space.str_w(self.getcode().co_names_w[index])
+        return self.space.text_w(self.getcode().co_names_w[index])
 
     def getname_w(self, index):
         return self.getcode().co_names_w[index]
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -97,6 +97,8 @@
 
     def str_w(self, s):
         return str(s)
+    def text_w(self, s):
+        return self.str_w(s)
 
     def len(self, x):
         return len(x)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -31,8 +31,8 @@
             return True
         if self.user_overridden_class or w_other.user_overridden_class:
             return False
-        s1 = space.str_w(self)
-        s2 = space.str_w(w_other)
+        s1 = space.bytes_w(self)
+        s2 = space.bytes_w(w_other)
         if len(s2) > 1:
             return s1 is s2
         else:            # strings of len <= 1 are unique-ified
@@ -41,7 +41,7 @@
     def immutable_unique_id(self, space):
         if self.user_overridden_class:
             return None
-        s = space.str_w(self)
+        s = space.bytes_w(self)
         if len(s) > 1:
             uid = compute_unique_id(s)
         else:            # strings of len <= 1 are unique-ified
@@ -571,7 +571,7 @@
         w_obj = space.str(w_object)
         if space.is_w(w_stringtype, space.w_str):
             return w_obj  # XXX might be reworked when space.str() typechecks
-        value = space.str_w(w_obj)
+        value = space.bytes_w(w_obj)
         w_obj = space.allocate_instance(W_BytesObject, w_stringtype)
         W_BytesObject.__init__(w_obj, value)
         return w_obj
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -696,7 +696,7 @@
             value = space.int_w(w_obj, allow_conversion=False)
         elif space.isinstance_w(w_value, space.w_str):
             value, w_longval = _string_to_int_or_long(space, w_value,
-                                                      space.str_w(w_value))
+                                                      space.text_w(w_value))
         elif space.isinstance_w(w_value, space.w_unicode):
             from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
             string = unicode_to_decimal_w(space, w_value)
@@ -721,7 +721,7 @@
             s = unicode_to_decimal_w(space, w_value)
         else:
             try:
-                s = space.str_w(w_value)
+                s = space.text_w(w_value)
             except OperationError as e:
                 raise oefmt(space.w_TypeError,
                             "int() can't convert non-string with explicit "

From pypy.commits at gmail.com  Tue Nov  1 13:11:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 10:11:47 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: remove wrap calls from the
 __builtin__ module
Message-ID: <5818ccd3.85951c0a.b163e.d031@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88042:1bddaf521a03
Date: 2016-11-01 18:11 +0100
http://bitbucket.org/pypy/pypy/changeset/1bddaf521a03/

Log:	remove wrap calls from the __builtin__ module

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -101,7 +101,7 @@
         # this is obscure and slow
         space = self.space
         try:
-            w_builtin = space.getitem(w_globals, space.wrap('__builtins__'))
+            w_builtin = space.getitem(w_globals, space.newtext('__builtins__'))
         except OperationError as e:
             if not e.match(space, space.w_KeyError):
                 raise
@@ -114,7 +114,7 @@
                 return w_builtin
         # no builtin! make a default one.  Give them None, at least.
         builtin = module.Module(space, None)
-        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
+        space.setitem(builtin.w_dict, space.newtext('None'), space.w_None)
         return builtin
 
     def setup_after_space_initialization(self):
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -20,7 +20,7 @@
     no __bases__ or if cls.__bases__ is not a tuple.
     """
     try:
-        w_bases = space.getattr(w_cls, space.wrap('__bases__'))
+        w_bases = space.getattr(w_cls, space.newtext('__bases__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -35,12 +35,12 @@
 
 def check_class(space, w_obj, msg):
     if not abstract_isclass_w(space, w_obj):
-        raise OperationError(space.w_TypeError, space.wrap(msg))
+        raise OperationError(space.w_TypeError, space.newtext(msg))
 
 
 def abstract_getclass(space, w_obj):
     try:
-        return space.getattr(w_obj, space.wrap('__class__'))
+        return space.getattr(w_obj, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise       # propagate other errors
@@ -62,7 +62,7 @@
     check_class(space, w_cls, "isinstance() arg 2 must be a class, type,"
                               " or tuple of classes and types")
     try:
-        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+        w_abstractclass = space.getattr(w_inst, space.newtext('__class__'))
     except OperationError as e:
         if e.async(space):      # ignore most exceptions
             raise
@@ -77,7 +77,7 @@
     if space.isinstance_w(w_inst, w_type):
         return True
     try:
-        w_abstractclass = space.getattr(w_inst, space.wrap('__class__'))
+        w_abstractclass = space.getattr(w_inst, space.newtext('__class__'))
     except OperationError as e:
         if e.async(space):      # ignore most exceptions
             raise
@@ -235,14 +235,14 @@
 another class.  When using a tuple as the second argument, check whether
 'cls' is a subclass of any of the classes listed in the tuple."""
     result = abstract_issubclass_w(space, w_cls, w_klass_or_tuple, True)
-    return space.wrap(result)
+    return space.newbool(result)
 
 def isinstance(space, w_obj, w_klass_or_tuple):
     """Check whether an object is an instance of a class (or of a subclass
 thereof).  When using a tuple as the second argument, check whether 'obj'
 is an instance of any of the classes listed in the tuple."""
     result = abstract_isinstance_w(space, w_obj, w_klass_or_tuple, True)
-    return space.wrap(result)
+    return space.newbool(result)
 
 # avoid namespace pollution
 app_issubclass = issubclass; del issubclass
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -39,12 +39,11 @@
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
         ast_node = ast.mod.from_object(space, w_source)
-        code = ec.compiler.compile_ast(ast_node, filename, mode, flags)
-        return space.wrap(code)
+        return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
     if space.isinstance_w(w_source, space.w_unicode):
         w_utf_8_source = space.call_method(w_source, "encode",
-                                           space.wrap("utf-8"))
+                                           space.newtext("utf-8"))
         source = space.str_w(w_utf_8_source)
         # This flag tells the parser to reject any coding cookies it sees.
         flags |= consts.PyCF_SOURCE_IS_UTF8
@@ -60,8 +59,7 @@
         node = ec.compiler.compile_to_ast(source, filename, mode, flags)
         return node.to_object(space)
     else:
-        code = ec.compiler.compile(source, filename, mode, flags)
-        return space.wrap(code)
+        return ec.compiler.compile(source, filename, mode, flags)
 
 
 def eval(space, w_code, w_globals=None, w_locals=None):
@@ -75,7 +73,7 @@
         space.isinstance_w(w_code, space.w_unicode)):
         w_code = compile(space,
                          space.call_method(w_code, 'lstrip',
-                                           space.wrap(' \t')),
+                                           space.newtext(' \t')),
                          "<string>", "eval")
 
     if not isinstance(w_code, PyCode):
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -28,7 +28,7 @@
             objtype_name = "<%s object>" % self.w_objtype.getname(space)
         else:
             objtype_name = 'NULL'
-        return space.wrap("<super: <class '%s'>, %s>" % (
+        return space.newtext("<super: <class '%s'>, %s>" % (
             self.w_starttype.getname(space), objtype_name))
 
     def get(self, space, w_obj, w_type=None):
@@ -77,7 +77,7 @@
         return w_objtype
 
     try:
-        w_type = space.getattr(w_obj_or_type, space.wrap('__class__'))
+        w_type = space.getattr(w_obj_or_type, space.newtext('__class__'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -130,12 +130,12 @@
         # our __doc__ comes from the getter if we don't have an explicit one
         if (space.is_w(self.w_doc, space.w_None) and
             not space.is_w(self.w_fget, space.w_None)):
-            w_getter_doc = space.findattr(self.w_fget, space.wrap('__doc__'))
+            w_getter_doc = space.findattr(self.w_fget, space.newtext('__doc__'))
             if w_getter_doc is not None:
                 if type(self) is W_Property:
                     self.w_doc = w_getter_doc
                 else:
-                    space.setattr(self, space.wrap('__doc__'), w_getter_doc)
+                    space.setattr(self, space.newtext('__doc__'), w_getter_doc)
                 self.getter_doc = True
 
     def get(self, space, w_obj, w_objtype=None):
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -54,7 +54,7 @@
 get a list in decending order."""
 
     if w_y is None:
-        w_start = space.wrap(0)
+        w_start = space.newint(0)
         w_stop = w_x
     else:
         w_start = w_x
@@ -91,7 +91,7 @@
     res_w = [None] * howmany
     v = start
     for idx in range(howmany):
-        res_w[idx] = space.wrap(v)
+        res_w[idx] = space.newint(v)
         v += step
     return space.newlist(res_w)
 
@@ -264,10 +264,10 @@
 
         self = space.allocate_instance(W_Enumerate, w_subtype)
         self.__init__(w_iter, start, w_start)
-        return space.wrap(self)
+        return self
 
     def descr___iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_next(self, space):
         from pypy.objspace.std.listobject import W_ListObject
@@ -289,14 +289,14 @@
                 try:
                     newval = rarithmetic.ovfcheck(index + 1)
                 except OverflowError:
-                    w_index = space.wrap(index)
-                    self.w_index = space.add(w_index, space.wrap(1))
+                    w_index = space.newint(index)
+                    self.w_index = space.add(w_index, space.newint(1))
                     self.index = -1
                 else:
                     self.index = newval
-            w_index = space.wrap(index)
+            w_index = space.newint(index)
         else:
-            self.w_index = space.add(w_index, space.wrap(1))
+            self.w_index = space.add(w_index, space.newint(1))
         if w_item is None:
             w_item = space.next(self.w_iter_or_list)
         return space.newtuple([w_index, w_item])
@@ -308,7 +308,7 @@
         w_new_inst = mod.get('enumerate_new')
         w_index = self.w_index
         if w_index is None:
-            w_index = space.wrap(self.index)
+            w_index = space.newint(self.index)
         w_info = space.newtuple([self.w_iter_or_list, w_index])
         return space.newtuple([w_new_inst, w_info])
 
@@ -319,7 +319,7 @@
         w_index = None
     else:
         index = -1
-    return space.wrap(W_Enumerate(w_iter_or_list, index, w_index))
+    return W_Enumerate(w_iter_or_list, index, w_index)
 
 W_Enumerate.typedef = TypeDef("enumerate",
     __new__=interp2app(W_Enumerate.descr___new__.im_func),
@@ -333,14 +333,14 @@
     """Return a iterator that yields items of sequence in reverse."""
     w_reversed = None
     if space.is_oldstyle_instance(w_sequence):
-        w_reversed = space.findattr(w_sequence, space.wrap("__reversed__"))
+        w_reversed = space.findattr(w_sequence, space.newtext("__reversed__"))
     else:
         w_reversed_descr = space.lookup(w_sequence, "__reversed__")
         if w_reversed_descr is not None:
             w_reversed = space.get(w_reversed_descr, w_sequence)
     if w_reversed is not None:
         return space.call_function(w_reversed)
-    return space.wrap(W_ReversedIterator(space, w_sequence))
+    return W_ReversedIterator(space, w_sequence)
 
 
 class W_ReversedIterator(W_Root):
@@ -352,14 +352,14 @@
         self.w_sequence = w_sequence
 
     def descr___iter__(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_length(self, space):
-        return space.wrap(0 if self.remaining == -1 else self.remaining + 1)
+        return space.newint(0 if self.remaining == -1 else self.remaining + 1)
 
     def descr_next(self, space):
         if self.remaining >= 0:
-            w_index = space.wrap(self.remaining)
+            w_index = space.newint(self.remaining)
             try:
                 w_item = space.getitem(self.w_sequence, w_index)
             except OperationError as e:
@@ -385,7 +385,7 @@
         mod      = space.interp_w(MixedModule, w_mod)
         w_new_inst = mod.get('reversed_new')
         w_seq = space.w_None if self.w_sequence is None else self.w_sequence
-        info_w = [w_seq, space.wrap(self.remaining)]
+        info_w = [w_seq, space.newint(self.remaining)]
         w_info = space.newtuple(info_w)
         return space.newtuple([w_new_inst, w_info])
 
@@ -407,7 +407,7 @@
     else:
         iterator.w_sequence = w_seq
         iterator.remaining = space.int_w(w_remaining)
-    return space.wrap(iterator)
+    return iterator
 
 
 class W_XRange(W_Root):
@@ -431,9 +431,9 @@
         else:
             stop = space.int_w(w_stop)
         howmany = get_len_of_range(space, start, stop, step)
-        obj = space.allocate_instance(W_XRange, w_subtype)
-        W_XRange.__init__(obj, space, start, howmany, step, promote_step)
-        return space.wrap(obj)
+        w_obj = space.allocate_instance(W_XRange, w_subtype)
+        W_XRange.__init__(w_obj, space, start, howmany, step, promote_step)
+        return w_obj
 
     def descr_repr(self):
         if self.start == 0 and self.step == 1:
@@ -442,10 +442,10 @@
             s = "xrange(%d, %d)" % (self.start, self._get_stop())
         else:
             s = "xrange(%d, %d, %d)" %(self.start, self._get_stop(), self.step)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
     def descr_len(self):
-        return self.space.wrap(self.len)
+        return self.space.newint(self.len)
 
     @unwrap_spec(i='index')
     def descr_getitem(self, i):
@@ -455,31 +455,31 @@
         if i < 0:
             i += len
         if 0 <= i < len:
-            return space.wrap(self.start + i * self.step)
+            return space.newint(self.start + i * self.step)
         raise oefmt(space.w_IndexError, "xrange object index out of range")
 
     def descr_iter(self):
         if self.promote_step and self.step == 1:
             stop = self.start + self.len
-            return self.space.wrap(W_XRangeStepOneIterator(self.space,
-                                                           self.start,
-                                                           stop))
+            return W_XRangeStepOneIterator(self.space,
+                                           self.start,
+                                           stop)
         else:
-            return self.space.wrap(W_XRangeIterator(self.space, self.start,
-                                                    self.len, self.step))
+            return W_XRangeIterator(self.space, self.start,
+                                    self.len, self.step)
 
     def descr_reversed(self):
         last = self.start + (self.len - 1) * self.step
-        return self.space.wrap(W_XRangeIterator(self.space, last, self.len,
-                                                -self.step))
+        return W_XRangeIterator(self.space, last, self.len,
+                                -self.step)
 
     def descr_reduce(self):
         space = self.space
         return space.newtuple(
             [space.type(self),
-             space.newtuple([space.wrap(self.start),
-                             space.wrap(self._get_stop()),
-                             space.wrap(self.step)])
+             space.newtuple([space.newint(self.start),
+                             space.newint(self._get_stop()),
+                             space.newint(self.step)])
              ])
 
     def _get_stop(self):
@@ -512,7 +512,7 @@
         self.step = step
 
     def descr_iter(self):
-        return self.space.wrap(self)
+        return self
 
     def descr_next(self):
         return self.next()
@@ -522,11 +522,11 @@
             item = self.current
             self.current = item + self.step
             self.remaining -= 1
-            return self.space.wrap(item)
+            return self.space.newint(item)
         raise OperationError(self.space.w_StopIteration, self.space.w_None)
 
     def descr_len(self):
-        return self.space.wrap(self.get_remaining())
+        return self.space.newint(self.get_remaining())
 
     def descr_reduce(self):
         from pypy.interpreter.mixedmodule import MixedModule
@@ -534,10 +534,9 @@
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('xrangeiter_new')
-        w        = space.wrap
         nt = space.newtuple
 
-        tup = [w(self.current), w(self.get_remaining()), w(self.step)]
+        tup = [space.newint(self.current), space.newint(self.get_remaining()), space.newint(self.step)]
         return nt([new_inst, nt(tup)])
 
     def get_remaining(self):
@@ -565,7 +564,7 @@
         if self.current < self.stop:
             item = self.current
             self.current = item + 1
-            return self.space.wrap(item)
+            return self.space.newint(item)
         raise OperationError(self.space.w_StopIteration, self.space.w_None)
 
     def get_remaining(self):
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -21,8 +21,8 @@
     if not space.isinstance_w(w_dict, space.w_dict):
         raise_type_err(space, 'bases', 'tuple', w_bases)
 
-    if not space.contains_w(w_dict, space.wrap("__doc__")):
-        space.setitem(w_dict, space.wrap("__doc__"), space.w_None)
+    if not space.contains_w(w_dict, space.newtext("__doc__")):
+        space.setitem(w_dict, space.newtext("__doc__"), space.w_None)
 
     # XXX missing: lengthy and obscure logic about "__module__"
 
@@ -109,7 +109,7 @@
             if name == "__dict__":
                 return self.w_dict
             elif name == "__name__":
-                return space.wrap(self.name)
+                return space.newtext(self.name)
             elif name == "__bases__":
                 return space.newtuple(self.bases_w)
         w_value = self.lookup(space, name)
@@ -138,7 +138,7 @@
                 if not self.has_user_del(space):
                     msg = ("a __del__ method added to an existing class will "
                            "only be called on instances made from now on")
-                    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+                    space.warn(space.newtext(msg), space.w_RuntimeWarning)
         space.setitem(self.w_dict, w_attr, w_value)
 
     def descr_delattr(self, space, w_attr):
@@ -161,9 +161,9 @@
     def descr_str(self, space):
         mod = self.get_module_string(space)
         if mod == "?":
-            return space.wrap(self.name)
+            return space.newtext(self.name)
         else:
-            return space.wrap("%s.%s" % (mod, self.name))
+            return space.newtext("%s.%s" % (mod, self.name))
 
     def get_module_string(self, space):
         try:
@@ -338,7 +338,7 @@
         w_meth = self.getattr_from_class(space, '__getattr__')
         if w_meth is not None:
             try:
-                return space.call_function(w_meth, space.wrap(name))
+                return space.call_function(w_meth, space.newtext(name))
             except OperationError as e:
                 if not exc and e.match(space, space.w_AttributeError):
                     return None     # eat the AttributeError
@@ -375,7 +375,7 @@
                     and self.getdictvalue(space, '__del__') is None):
                     msg = ("a __del__ method added to an instance with no "
                            "__del__ in the class will not be called")
-                    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+                    space.warn(space.newtext(msg), space.w_RuntimeWarning)
         if w_meth is not None:
             space.call_function(w_meth, w_name, w_value)
         else:
@@ -433,14 +433,14 @@
             if space.len_w(w_format_spec) > 0:
                 msg = ("object.__format__ with a non-empty format string is "
                        "deprecated")
-                space.warn(space.wrap(msg), space.w_PendingDeprecationWarning)
+                space.warn(space.newtext(msg), space.w_PendingDeprecationWarning)
             return space.format(w_as_str, w_format_spec)
 
     def descr_len(self, space):
         w_meth = self.getattr(space, '__len__')
         w_result = space.call_function(w_meth)
         if space.isinstance_w(w_result, space.w_int):
-            if space.is_true(space.lt(w_result, space.wrap(0))):
+            if space.is_true(space.lt(w_result, space.newint(0))):
                 raise oefmt(space.w_ValueError, "__len__() should return >= 0")
             return w_result
         raise oefmt(space.w_TypeError, "__len__() should return an int")
@@ -502,7 +502,7 @@
                 return space.w_True
         w_result = space.call_function(w_func)
         if space.isinstance_w(w_result, space.w_int):
-            if space.is_true(space.lt(w_result, space.wrap(0))):
+            if space.is_true(space.lt(w_result, space.newint(0))):
                 raise oefmt(space.w_ValueError,
                             "__nonzero__() should return >= 0")
             return w_result
@@ -527,10 +527,10 @@
                                     "__cmp__ must return int")
                     raise
                 if res > 0:
-                    return space.wrap(1)
+                    return space.newint(1)
                 if res < 0:
-                    return space.wrap(-1)
-                return space.wrap(0)
+                    return space.newint(-1)
+                return space.newint(0)
         if isinstance(w_b, W_InstanceObject):
             w_func = w_b.getattr(space, '__cmp__', False)
             if w_func is not None:
@@ -545,10 +545,10 @@
                                     "__cmp__ must return int")
                     raise
                 if res < 0:
-                    return space.wrap(1)
+                    return space.newint(1)
                 if res > 0:
-                    return space.wrap(-1)
-                return space.wrap(0)
+                    return space.newint(-1)
+                return space.newint(0)
         return space.w_NotImplemented
 
     def descr_hash(self, space):
@@ -559,7 +559,7 @@
             if w_eq is not None or w_cmp is not None:
                 raise oefmt(space.w_TypeError, "unhashable instance")
             else:
-                return space.wrap(compute_identity_hash(self))
+                return space.newint(compute_identity_hash(self))
         w_ret = space.call_function(w_func)
         if (not space.isinstance_w(w_ret, space.w_int) and
             not space.isinstance_w(w_ret, space.w_long)):
@@ -598,7 +598,7 @@
     def descr_contains(self, space, w_obj):
         w_func = self.getattr(space, '__contains__', False)
         if w_func is not None:
-            return space.wrap(space.is_true(space.call_function(w_func, w_obj)))
+            return space.newbool(space.is_true(space.call_function(w_func, w_obj)))
         # now do it ourselves
         w_iter = space.iter(self)
         while 1:
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -30,7 +30,7 @@
         c = UNICHR(code)
     except ValueError:
         raise oefmt(space.w_ValueError, "unichr() arg out of range")
-    return space.wrap(c)
+    return space.newunicode(c)
 
 def len(space, w_obj):
     "len(object) -> integer\n\nReturn the number of items of a sequence or mapping."
@@ -46,7 +46,7 @@
     # unmodified (and not e.g. unwrapped-rewrapped).
     if not space.is_w(space.type(w_name), space.w_str):
         name = space.str_w(w_name)    # typecheck
-        w_name = space.wrap(name)     # rewrap as a real string
+        w_name = space.newtext(name)     # rewrap as a real string
     return w_name
 
 def delattr(space, w_object, w_name):
@@ -135,23 +135,23 @@
 
     # nans, infinities and zeros round to themselves
     if number == 0 or isinf(number) or isnan(number):
-        return space.wrap(number)
+        return space.newfloat(number)
 
     # Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
     # always rounds to itself.  For ndigits < NDIGITS_MIN, x always
     # rounds to +-0.0.
     if ndigits > NDIGITS_MAX:
-        return space.wrap(number)
+        return space.newfloat(number)
     elif ndigits < NDIGITS_MIN:
         # return 0.0, but with sign of x
-        return space.wrap(0.0 * number)
+        return space.newfloat(0.0 * number)
 
     # finite x, and ndigits is not unreasonably large
     z = round_double(number, ndigits)
     if isinf(z):
         raise oefmt(space.w_OverflowError,
                     "rounded value too large to represent")
-    return space.wrap(z)
+    return space.newfloat(z)
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Tue Nov  1 13:13:38 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 10:13:38 -0700 (PDT)
Subject: [pypy-commit] pypy openssl-1.1: Skip running this new test if
 ssl.get_default_verify_paths() fails (env
Message-ID: <5818cd42.8ca71c0a.ff099.df63@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: openssl-1.1
Changeset: r88043:5f122f991fba
Date: 2016-11-01 18:09 +0100
http://bitbucket.org/pypy/pypy/changeset/5f122f991fba/

Log:	Skip running this new test if ssl.get_default_verify_paths() fails
	(env not set up)

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -273,6 +273,8 @@
     def test_peer_certificate_verify(self):
         import _ssl, ssl, gc
         paths = ssl.get_default_verify_paths()
+        if not paths.capath and not paths.cafile:
+            skip("ssl.get_default_verify_paths() failed to return any path")
 
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.verify_mode = _ssl.CERT_REQUIRED

From pypy.commits at gmail.com  Tue Nov  1 13:49:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 10:49:47 -0700 (PDT)
Subject: [pypy-commit] pypy openssl-1.1: We need 'OPENSSL_free' on 1.1,
 and we need it to be a macro there
Message-ID: <5818d5bb.e8edc20a.9a8b.a0d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: openssl-1.1
Changeset: r88044:9bf5a9ff7207
Date: 2016-11-01 18:48 +0100
http://bitbucket.org/pypy/pypy/changeset/9bf5a9ff7207/

Log:	We need 'OPENSSL_free' on 1.1, and we need it to be a macro there

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -499,10 +499,12 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-libssl_OPENSSL_free = external(
-    'OPENSSL_free' if OPENSSL_VERSION_NUMBER >= 0x10100000
-    else 'CRYPTO_free',
-    [rffi.VOIDP], lltype.Void)
+if OPENSSL_VERSION_NUMBER >= 0x10100000:
+    ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
+else:
+    ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
+    libssl_OPENSSL_free = libssl_CRYPTO_free
+    del libssl_CRYPTO_free
 
 ssl_external('SSL_write', [SSL, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=SAVE_ERR)

From pypy.commits at gmail.com  Tue Nov  1 14:00:09 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Nov 2016 11:00:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <5818d829.d4301c0a.ba28a.eaa6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88046:22a3e8d96787
Date: 2016-11-01 17:59 +0000
http://bitbucket.org/pypy/pypy/changeset/22a3e8d96787/

Log:	merge heads

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,10 +1,11 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
 from rpython.rlib.runicode import code_to_unichr, MAXUNICODE
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.module.unicodedata import unicodedb
 
 
 class VersionTag(object):
@@ -295,6 +296,26 @@
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
 
+def namereplace_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        builder = StringBuilder()
+        pos = start
+        while pos < end:
+            oc = ord(obj[pos])
+            builder.append('\\N{')
+            builder.append(unicodedb.name(oc))
+            builder.append('}')
+            pos += 1
+        return space.newtuple([space.newbytes(builder.build()), w_end])
+    else:
+        raise oefmt(space.w_TypeError,
+                    "don't know how to handle %T in error callback", w_exc)
+
 def surrogatepass_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
@@ -382,7 +403,8 @@
     "NOT_RPYTHON"
     state = space.fromcache(CodecState)
     for error in ("strict", "ignore", "replace", "xmlcharrefreplace",
-                  "backslashreplace", "surrogateescape", "surrogatepass"):
+                  "backslashreplace", "surrogateescape", "surrogatepass",
+                  "namereplace"):
         name = error + "_errors"
         state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
@@ -667,7 +689,7 @@
             return errorchar
 
         raise oefmt(space.w_TypeError,
-            "character mapping must return integer, None or unicode")
+            "character mapping must return integer, None or str")
 
 class Charmap_Encode:
     def __init__(self, space, w_mapping):
@@ -700,7 +722,7 @@
             return errorchar
 
         raise oefmt(space.w_TypeError,
-            "character mapping must return integer, None or str")
+            "character mapping must return integer, bytes or None, not str")
 
 
 @unwrap_spec(string='bufferstr', errors='str_or_None')
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -50,7 +50,7 @@
                     ]
         for s in insecure:
             buf = b"S" + s + b"\012p0\012."
-            raises (ValueError, pickle.loads, buf)
+            raises ((ValueError, pickle.UnpicklingError), pickle.loads, buf)
 
     def test_unicodedecodeerror(self):
         assert str(UnicodeDecodeError(
@@ -112,7 +112,7 @@
         assert charmap_decode(b'xxx\xff', 'strict', map) == ('xxx\xff', 4)
 
         exc = raises(TypeError, charmap_decode, b'\xff', "strict",  {0xff: b'a'})
-        assert str(exc.value) == "character mapping must return integer, None or unicode"
+        assert str(exc.value) == "character mapping must return integer, None or str"
         raises(TypeError, charmap_decode, b'\xff', "strict",  {0xff: 0x110000})
         assert (charmap_decode(b"\x00\x01\x02", "strict",
                                {0: 0x10FFFF, 1: ord('b'), 2: ord('c')}) ==
@@ -561,9 +561,14 @@
         assert b'\xff'.decode('utf-7', 'ignore') == ''
         assert b'\x00'.decode('unicode-internal', 'ignore') == ''
 
-    def test_backslahreplace(self):
+    def test_backslashreplace(self):
         assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000'
 
+    def test_namereplace(self):
+        assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'namereplace') == (
+            b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}'
+            b'\\N{CJK UNIFIED IDEOGRAPH-8000}')
+
     def test_surrogateescape(self):
         assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b'
         assert 'a\udc80b'.encode('utf-8', 'surrogateescape') == b'a\x80b'
@@ -682,7 +687,7 @@
         exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: 300})
         assert str(exc.value) == 'character mapping must be in range(256)'
         exc = raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: u'a'})
-        assert str(exc.value) == 'character mapping must return integer, None or str'
+        assert str(exc.value) == 'character mapping must return integer, bytes or None, not str'
         raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None})
 
     def test_charmap_encode_replace(self):

From pypy.commits at gmail.com  Tue Nov  1 14:00:07 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 01 Nov 2016 11:00:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix memoryview slicing and remove
 confusing _apply_itemsize()
Message-ID: <5818d827.4a0e1c0a.27e88.ed08@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88045:dad399a5cc08
Date: 2016-11-01 17:50 +0000
http://bitbucket.org/pypy/pypy/changeset/dad399a5cc08/

Log:	Fix memoryview slicing and remove confusing _apply_itemsize()

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -728,6 +728,7 @@
     # Unicode with encoding
     if space.isinstance_w(w_source, space.w_unicode):
         if encoding is None:
+            raise RuntimeError
             raise oefmt(space.w_TypeError,
                         "string argument without an encoding")
         from pypy.objspace.std.unicodeobject import encode_object
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -259,40 +259,38 @@
 
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
-        itemsize = self.getitemsize()
-        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
+            itemsize = self.getitemsize()
             if itemsize == 1:
                 ch = self.buf.getitem(start)
                 return space.newint(ord(ch))
             else:
                 # TODO: this probably isn't very fast
-                buf = SubBuffer(self.buf, start, itemsize)
+                buf = SubBuffer(self.buf, start*itemsize, itemsize)
                 fmtiter = UnpackFormatIterator(space, buf)
                 fmtiter.interpret(self.format)
                 return fmtiter.result_w[0]
         elif step == 1:
             mv = W_MemoryView.copy(self)
-            mv.slice(start, stop, step, size)
+            mv.slice(start, step, size)
             mv._init_flags()
             return mv
         else:
             mv = W_MemoryView.copy(self)
-            mv.slice(start, stop, step, size)
+            mv.slice(start, step, size)
             mv.length = mv.bytecount_from_shape()
             mv._init_flags()
             return mv
 
-    def slice(self, start, stop, step, size):
+    def slice(self, start, step, size):
         # modifies the buffer, shape and stride to allow step to be > 1
-        # NOTE that start, stop & size are already byte offsets/count
         # TODO subbuffer
         strides = self.getstrides()[:]
         shape = self.getshape()[:]
         itemsize = self.getitemsize()
         dim = 0
-        self.buf = SubBuffer(self.buf, strides[dim] * (start//itemsize), size*step)
-        shape[dim] = size//itemsize
+        self.buf = SubBuffer(self.buf, strides[dim] * start, size*step*itemsize)
+        shape[dim] = size
         strides[dim] = strides[dim] * step
         self.strides = strides
         self.shape = shape
@@ -313,18 +311,6 @@
         return W_MemoryView(buf, view.getformat(), view.getitemsize(),
                             view.getndim(), view.getshape()[:], view.getstrides()[:])
 
-    def _apply_itemsize(self, space, start, size, itemsize):
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-
-        stop  = start + size
-        # start & stop are now byte offset, thus use self.buf.getlength()
-        if stop > self.buf.getlength():
-            raise oefmt(space.w_IndexError, 'index out of range')
-
-        return start, stop, size
-
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.buf.readonly:
@@ -333,7 +319,6 @@
             raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         itemsize = self.getitemsize()
-        start, stop, size = self._apply_itemsize(space, start, size, itemsize)
         if step == 0:  # index only
             if itemsize == 1:
                 ch = getbytevalue(space, w_obj)
@@ -347,13 +332,13 @@
                     raise oefmt(space.w_TypeError,
                                 "memoryview: invalid type for format '%s'",
                                 self.format)
-                self.buf.setslice(start, fmtiter.result.build())
+                self.buf.setslice(start * itemsize, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != size:
+            if value.getlength() != size * itemsize:
                 raise oefmt(space.w_ValueError,
                             "cannot modify size of memoryview object")
-            self.buf.setslice(start, value.as_str())
+            self.buf.setslice(start * itemsize, value.as_str())
         else:
             if self.getndim() != 1:
                 raise oefmt(space.w_NotImplementedError,
@@ -366,11 +351,11 @@
             src = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             dst_strides = self.getstrides()
             dim = 0
-            dst = SubBuffer(self.buf, start, size)
+            dst = SubBuffer(self.buf, start * itemsize, size * itemsize)
             src_stride0 = dst_strides[dim]
 
             off = 0
-            src_shape0 = size // itemsize
+            src_shape0 = size
             src_stride0 = src.getstrides()[0]
             if isinstance(w_obj, W_MemoryView):
                 src_stride0 = w_obj.getstrides()[0]
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -47,6 +47,8 @@
         assert list(w) == [97]
         v[::2] = b'ABC'
         assert data == bytearray(eval("b'AbBeCg'"))
+        assert v[::2] == b'ABC'
+        assert v[::-2] == b'geb'
 
     def test_memoryview_attrs(self):
         v = memoryview(b"a"*100)

From pypy.commits at gmail.com  Tue Nov  1 14:16:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 11:16:23 -0700 (PDT)
Subject: [pypy-commit] pypy openssl-1.1: some blind LibreSSL fixes
Message-ID: <5818dbf7.44941c0a.3f33e.e65b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: openssl-1.1
Changeset: r88047:a11c8673e565
Date: 2016-11-01 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/a11c8673e565/

Log:	some blind LibreSSL fixes

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -59,7 +59,7 @@
         '#define pypy_GENERAL_NAME_pop_free(names) (sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free))',
         '#define pypy_DIST_POINT_fullname(obj) (obj->distpoint->name.fullname)',
         # Backwards compatibility for functions introduced in 1.1
-        '#if (OPENSSL_VERSION_NUMBER < 0x10100000)\n'
+        '#if (OPENSSL_VERSION_NUMBER < 0x10100000) || defined(LIBRESSL_VERSION_NUMBER)\n'
         '#  define COMP_get_name(meth) (meth->name)\n'
         '#  define COMP_get_type(meth) (meth->type)\n'
         '#  define X509_NAME_ENTRY_set(ne) (ne->set)\n'
@@ -67,9 +67,9 @@
         '#  define X509_OBJECT_get_type(obj) (obj->type)\n'
         '#  define X509_STORE_get0_objects(store) (store->objs)\n'
         '#  define X509_STORE_get0_param(store) (store->param)\n'
-        '#else /* (OPENSSL_VERSION_NUMBER < 0x10100000) */\n'
+        '#else\n'
         '#  define OPENSSL_NO_SSL2\n'
-        '#endif /* (OPENSSL_VERSION_NUMBER < 0x10100000) */',
+        '#endif',
     ],
 )
 
@@ -108,6 +108,7 @@
     ASN1_ITEM_EXP = ASN1_ITEM
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
 LIBRESSL = cconfig["LIBRESSL"]
+OPENSSL_1_1 = OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
 
@@ -137,7 +138,7 @@
     SSL_OP_SINGLE_ECDH_USE = rffi_platform.ConstantInteger(
         "SSL_OP_SINGLE_ECDH_USE")
     SSL_OP_NO_COMPRESSION = rffi_platform.DefinedConstantInteger(
-         "SSL_OP_NO_COMPRESSION")
+        "SSL_OP_NO_COMPRESSION")
     SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger(
         "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS")
     SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger(
@@ -283,9 +284,9 @@
         name, argtypes, restype, **kw)
 
 ssl_external('SSL_load_error_strings', [], lltype.Void,
-    macro=bool(OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL) or None)
+    macro=OPENSSL_1_1 or None)
 ssl_external('SSL_library_init', [], rffi.INT,
-    macro=bool(OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL) or None)
+    macro=OPENSSL_1_1 or None)
 ssl_external('CRYPTO_num_locks', [], rffi.INT)
 ssl_external('CRYPTO_set_locking_callback',
              [lltype.Ptr(lltype.FuncType(
@@ -314,8 +315,7 @@
 # Windows note: fails in untranslated tests if the following function is
 # made 'macro=True'.  Not sure I want to dig into the reason for that mess.
 libssl_TLS_method = external(
-    'TLS_method' if OPENSSL_VERSION_NUMBER >= 0x10100000
-    else 'SSLv23_method',
+    'TLS_method' if OPENSSL_1_1 else 'SSLv23_method',
     [], SSL_METHOD)
 ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=rffi.RFFI_FULL_ERRNO_ZERO)
@@ -345,7 +345,7 @@
              lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tmp_ecdh', [SSL_CTX, EC_KEY], lltype.Void, macro=True)
-if 0x10100000 > OPENSSL_VERSION_NUMBER >= 0x10002000:
+if OPENSSL_VERSION_NUMBER >= 0x10002000 and not OPENSSL_1_1:
     ssl_external('SSL_CTX_set_ecdh_auto', [SSL_CTX, rffi.INT], lltype.Void,
                  macro=True)
 else:
@@ -392,7 +392,7 @@
 ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT)
 ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING)
 ssl_external('X509_NAME_ENTRY_set', [X509_NAME_ENTRY], rffi.INT,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+    macro=(not OPENSSL_1_1) or None)
 ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT, save_err=SAVE_ERR)
 ssl_external('X509_free', [X509], lltype.Void, releasegil=False)
 ssl_external('X509_check_ca', [X509], rffi.INT)
@@ -411,9 +411,9 @@
 ssl_external('X509_VERIFY_PARAM_clear_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_STORE_add_cert', [X509_STORE, X509], rffi.INT)
 ssl_external('X509_STORE_get0_objects', [X509_STORE], stack_st_X509_OBJECT,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+    macro=bool(not OPENSSL_1_1) or None)
 ssl_external('X509_STORE_get0_param', [X509_STORE], X509_VERIFY_PARAM,
-    macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+    macro=bool(not OPENSSL_1_1) or None)
 
 ssl_external('X509_get_default_cert_file_env', [], rffi.CCHARP)
 ssl_external('X509_get_default_cert_file', [], rffi.CCHARP)
@@ -451,13 +451,13 @@
 ssl_external('sk_X509_OBJECT_value', [stack_st_X509_OBJECT, rffi.INT],
              X509_OBJECT, macro=True)
 ssl_external('X509_OBJECT_get0_X509', [X509_OBJECT], X509,
-             macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('X509_OBJECT_get_type', [X509_OBJECT], rffi.INT,
-             macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('COMP_get_name', [COMP_METHOD], rffi.CCHARP,
-             macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('COMP_get_type', [COMP_METHOD], rffi.INT,
-             macro=bool(OPENSSL_VERSION_NUMBER < 0x10100000) or None)
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('sk_DIST_POINT_num', [stack_st_DIST_POINT], rffi.INT,
              macro=True)
 ssl_external('sk_DIST_POINT_value', [stack_st_DIST_POINT, rffi.INT], DIST_POINT,
@@ -499,7 +499,7 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if OPENSSL_1_1:
     ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
 else:
     ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
@@ -572,7 +572,7 @@
 
 OpenSSL_add_all_digests = external(
     'OpenSSL_add_all_digests', [], lltype.Void,
-    macro=bool(OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL) or None)
+    macro=OPENSSL_1_1 or None)
 EVP_get_digestbyname = external(
     'EVP_get_digestbyname',
     [rffi.CCHARP], EVP_MD)
@@ -592,15 +592,13 @@
 EVP_MD_CTX_copy = external(
     'EVP_MD_CTX_copy', [EVP_MD_CTX, EVP_MD_CTX], rffi.INT)
 EVP_MD_CTX_new = external(
-    'EVP_MD_CTX_new' if OPENSSL_VERSION_NUMBER >= 0x10100000
-    else 'EVP_MD_CTX_create',
+    'EVP_MD_CTX_new' if OPENSSL_1_1 else 'EVP_MD_CTX_create',
     [], EVP_MD_CTX)
 EVP_MD_CTX_free = external(
-    'EVP_MD_CTX_free' if OPENSSL_VERSION_NUMBER >= 0x10100000
-    else 'EVP_MD_CTX_destroy',
+    'EVP_MD_CTX_free' if OPENSSL_1_1 else 'EVP_MD_CTX_destroy',
     [EVP_MD_CTX], lltype.Void, releasegil=False)
 
-if OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL:
+if OPENSSL_1_1:
     PKCS5_PBKDF2_HMAC = external('PKCS5_PBKDF2_HMAC', [
         rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT, rffi.INT, EVP_MD,
         rffi.INT, rffi.CCHARP], rffi.INT)

From pypy.commits at gmail.com  Tue Nov  1 14:34:35 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 11:34:35 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: remove space.wrap from the array
 module (one left)
Message-ID: <5818e03b.e271c20a.60f81.b7e5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88048:fea7fb1fd7fb
Date: 2016-11-01 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/fea7fb1fd7fb/

Log:	remove space.wrap from the array module (one left)

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -53,11 +53,11 @@
 
 
 def descr_itemsize(space, self):
-    return space.wrap(self.itemsize)
+    return space.newint(self.itemsize)
 
 
 def descr_typecode(space, self):
-    return space.wrap(self.typecode)
+    return space.newint(self.typecode)
 
 arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
                               reds='auto')
@@ -265,14 +265,14 @@
             size = ovfcheck(self.itemsize * n)
         except OverflowError:
             raise MemoryError
-        w_item = space.call_method(w_f, 'read', space.wrap(size))
+        w_item = space.call_method(w_f, 'read', space.newint(size))
         item = space.bytes_w(w_item)
         if len(item) < size:
             n = len(item) % self.itemsize
             elems = max(0, len(item) - (len(item) % self.itemsize))
             if n != 0:
                 item = item[0:elems]
-            self.descr_fromstring(space, space.wrap(item))
+            self.descr_fromstring(space, space.newint(item))
             raise oefmt(space.w_EOFError, "not enough items in file")
         self.descr_fromstring(space, w_item)
 
@@ -315,7 +315,7 @@
         """
         if self.typecode == 'u':
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.wrap(rffi.wcharpsize2unicode(buf, self.len))
+            return space.newunicode(rffi.wcharpsize2unicode(buf, self.len))
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
@@ -328,8 +328,8 @@
         The length should be multiplied by the itemsize attribute to calculate
         the buffer length in bytes.
         """
-        w_ptr = space.wrap(self._buffer_as_unsigned())
-        w_len = space.wrap(self.len)
+        w_ptr = space.newint(self._buffer_as_unsigned())
+        w_len = space.newint(self.len)
         return space.newtuple([w_ptr, w_len])
 
     def descr_reduce(self, space):
@@ -337,11 +337,11 @@
         """
         if self.len > 0:
             w_s = self.descr_tostring(space)
-            args = [space.wrap(self.typecode), w_s]
+            args = [space.newtext(self.typecode), w_s]
         else:
-            args = [space.wrap(self.typecode)]
+            args = [space.newtext(self.typecode)]
         try:
-            w_dict = space.getattr(self, space.wrap('__dict__'))
+            w_dict = space.getattr(self, space.newtext('__dict__'))
         except OperationError:
             w_dict = space.w_None
         return space.newtuple([space.type(self), space.newtuple(args), w_dict])
@@ -382,7 +382,7 @@
         self._charbuf_stop()
 
     def descr_len(self, space):
-        return space.wrap(self.len)
+        return space.newint(self.len)
 
     def descr_eq(self, space, w_arr2):
         "x.__eq__(y) <==> x==y"
@@ -474,19 +474,19 @@
 
     def descr_repr(self, space):
         if self.len == 0:
-            return space.wrap("array('%s')" % self.typecode)
+            return space.newtext("array('%s')" % self.typecode)
         elif self.typecode == "c":
             r = space.repr(self.descr_tostring(space))
             s = "array('%s', %s)" % (self.typecode, space.str_w(r))
-            return space.wrap(s)
+            return space.newtext(s)
         elif self.typecode == "u":
             r = space.repr(self.descr_tounicode(space))
             s = "array('%s', %s)" % (self.typecode, space.str_w(r))
-            return space.wrap(s)
+            return space.newtext(s)
         else:
             r = space.repr(self.descr_tolist(space))
             s = "array('%s', %s)" % (self.typecode, space.str_w(r))
-            return space.wrap(s)
+            return space.newtext(s)
 
 W_ArrayBase.typedef = TypeDef(
     'array.array',
@@ -706,7 +706,7 @@
                     if not mytype.signed:
                         msg = 'un' + msg      # 'signed' => 'unsigned'
                     raise OperationError(self.space.w_OverflowError,
-                                         self.space.wrap(msg))
+                                         self.space.newtext(msg))
             return result
 
         @rgc.must_be_light_finalizer
@@ -845,9 +845,11 @@
             item = self.buffer[idx]
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
+                return space.newint(item)
             elif mytype.typecode == 'f':
                 item = float(item)
-            return space.wrap(item)
+                return space.newfloat(item)
+            return space.wrap(item) # YYY
 
         # interface
 
@@ -865,13 +867,13 @@
                 w_item = self.w_getitem(space, i)
                 if space.is_true(space.eq(w_item, w_val)):
                     cnt += 1
-            return space.wrap(cnt)
+            return space.newint(cnt)
 
         def descr_index(self, space, w_val):
             for i in range(self.len):
                 w_item = self.w_getitem(space, i)
                 if space.is_true(space.eq(w_item, w_val)):
-                    return space.wrap(i)
+                    return space.newint(i)
             raise oefmt(space.w_ValueError, "array.index(x): x not in list")
 
         def descr_reverse(self, space):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -202,6 +202,9 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    newtext = newbytes
+    newtext_or_none = newbytes
+
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):

From pypy.commits at gmail.com  Tue Nov  1 14:34:36 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 11:34:36 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: remove wraps in struct module
Message-ID: <5818e03c.ca13c20a.9a8b.b115@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88049:e725a8848ced
Date: 2016-11-01 19:04 +0100
http://bitbucket.org/pypy/pypy/changeset/e725a8848ced/

Log:	remove wraps in struct module

diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -15,7 +15,7 @@
 #      Stackless does use this so it might be needed here as well.
 
 def cell_new(space):
-    return space.wrap(instantiate(Cell))
+    return instantiate(Cell)
 
 def code_new(space, __args__):
     w_type = space.gettypeobject(PyCode.typedef)
@@ -24,11 +24,11 @@
 def func_new(space):
     fu = instantiate(Function)
     fu.w_func_dict = space.newdict()
-    return space.wrap(fu)
+    return fu
 
 def module_new(space, w_name, w_dict):
     new_mod = Module(space, w_name, w_dict, add_package=False)
-    return space.wrap(new_mod)
+    return new_mod
 
 def method_new(space, __args__):
     w_type = space.gettypeobject(Method.typedef)
@@ -57,21 +57,21 @@
 
 def frame_new(space):
     new_frame = instantiate(space.FrameClass)   # XXX fish
-    return space.wrap(new_frame)
+    return new_frame
 
 def traceback_new(space):
     tb = instantiate(PyTraceback)
-    return space.wrap(tb)
+    return tb
 
 def generator_new(space):
     new_generator = instantiate(GeneratorIterator)
-    return space.wrap(new_generator)
+    return new_generator
 
 @unwrap_spec(current=int, remaining=int, step=int)
 def xrangeiter_new(space, current, remaining, step):
     from pypy.module.__builtin__.functional import W_XRangeIterator
     new_iter = W_XRangeIterator(space, current, remaining, step)
-    return space.wrap(new_iter)
+    return new_iter
 
 @unwrap_spec(identifier=str)
 def builtin_code(space, identifier):
@@ -112,13 +112,12 @@
     a tuple with the positions of NULLs as first element.
     """
     tup = [None] * (len(seq_w) + 1)
-    w = space.wrap
     num = 1
     nulls = [None for i in seq_w if i is None]
     null_num = 0
     for w_obj in seq_w:
         if w_obj is None:
-            nulls[null_num] = w(num - 1)
+            nulls[null_num] = space.newint(num - 1)
             null_num += 1
             w_obj = space.w_None
         tup[num] = w_obj
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -78,7 +78,7 @@
                 else:
                     msg = "integer argument expected, got non-integer" \
                           " (implicit conversion using __int__ is deprecated)"
-                space.warn(space.wrap(msg), space.w_DeprecationWarning)
+                space.warn(space.newtext(msg), space.w_DeprecationWarning)
                 w_index = space.int(w_obj)   # wrapped float -> wrapped int or long
             if w_index is None:
                 raise StructError("cannot convert argument to integer")
@@ -148,7 +148,7 @@
 
     @specialize.argtype(1)
     def appendobj(self, value):
-        self.result_w.append(self.space.wrap(value))
+        self.result_w.append(self.space.wrap(value)) # YYY
 
     def get_pos(self):
         return self.pos
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -26,15 +26,15 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return fmtiter.totalsize
 
 
 @unwrap_spec(format=str)
 def calcsize(space, format):
-    return space.wrap(_calcsize(space, format))
+    return space.newint(_calcsize(space, format))
 
 
 def _pack(space, format, args_w):
@@ -46,15 +46,15 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return fmtiter.result.build()
 
 
 @unwrap_spec(format=str)
 def pack(space, format, args_w):
-    return space.wrap(_pack(space, format, args_w))
+    return space.newbytes(_pack(space, format, args_w))
 
 
 # XXX inefficient
@@ -77,9 +77,9 @@
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
-        raise OperationError(space.w_OverflowError, space.wrap(e.msg))
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
-        raise OperationError(get_error(space), space.wrap(e.msg))
+        raise OperationError(get_error(space), space.newtext(e.msg))
     return space.newtuple(fmtiter.result_w[:])
 
 

From pypy.commits at gmail.com  Tue Nov  1 14:34:39 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 11:34:39 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: itertools
Message-ID: <5818e03f.d5901c0a.3cc26.1005@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88051:3c3faf4c7f55
Date: 2016-11-01 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/3c3faf4c7f55/

Log:	itertools

diff --git a/pypy/module/errno/__init__.py b/pypy/module/errno/__init__.py
--- a/pypy/module/errno/__init__.py
+++ b/pypy/module/errno/__init__.py
@@ -22,5 +22,5 @@
 for name in dir(errno):
     if name.startswith('__') or name in Module.interpleveldefs:
         continue
-    Module.interpleveldefs[name] = ("space.wrap(%s)" %
+    Module.interpleveldefs[name] = ("space.newint(%s)" %
                                     (getattr(errno, name), ))
diff --git a/pypy/module/errno/interp_errno.py b/pypy/module/errno/interp_errno.py
--- a/pypy/module/errno/interp_errno.py
+++ b/pypy/module/errno/interp_errno.py
@@ -1,5 +1,5 @@
 import errno
 
 def get_errorcode(space):
-    return space.wrap(errno.errorcode)
+    return space.newint(errno.errorcode)
 
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -12,7 +12,7 @@
         self.w_step = w_step
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         w_c = self.w_c
@@ -22,7 +22,7 @@
     def single_argument(self):
         space = self.space
         return (space.isinstance_w(self.w_step, space.w_int) and
-                space.eq_w(self.w_step, space.wrap(1)))
+                space.eq_w(self.w_step, space.newint(1)))
 
     def repr_w(self):
         space = self.space
@@ -32,7 +32,7 @@
         else:
             step = space.str_w(space.repr(self.w_step))
             s = 'count(%s, %s)' % (c, step)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
     def reduce_w(self):
         space = self.space
@@ -54,7 +54,7 @@
     check_number(space, w_step)
     r = space.allocate_instance(W_Count, w_subtype)
     r.__init__(space, w_start, w_step)
-    return space.wrap(r)
+    return r
 
 W_Count.typedef = TypeDef(
         'itertools.count',
@@ -98,12 +98,12 @@
         return self.w_obj
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def length_w(self):
         if not self.counting:
             return self.space.w_NotImplemented
-        return self.space.wrap(self.count)
+        return self.space.newint(self.count)
 
     def repr_w(self):
         objrepr = self.space.str_w(self.space.repr(self.w_obj))
@@ -111,12 +111,12 @@
             s = 'repeat(%s, %d)' % (objrepr, self.count)
         else:
             s = 'repeat(%s)' % (objrepr,)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
 def W_Repeat___new__(space, w_subtype, w_object, w_times=None):
     r = space.allocate_instance(W_Repeat, w_subtype)
     r.__init__(space, w_object, w_times)
-    return space.wrap(r)
+    return r
 
 W_Repeat.typedef = TypeDef(
         'itertools.repeat',
@@ -151,7 +151,7 @@
         self.stopped = False
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.stopped:
@@ -168,7 +168,7 @@
 def W_TakeWhile___new__(space, w_subtype, w_predicate, w_iterable):
     r = space.allocate_instance(W_TakeWhile, w_subtype)
     r.__init__(space, w_predicate, w_iterable)
-    return space.wrap(r)
+    return r
 
 
 W_TakeWhile.typedef = TypeDef(
@@ -198,7 +198,7 @@
         self.started = False
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.started:
@@ -216,7 +216,7 @@
 def W_DropWhile___new__(space, w_subtype, w_predicate, w_iterable):
     r = space.allocate_instance(W_DropWhile, w_subtype)
     r.__init__(space, w_predicate, w_iterable)
-    return space.wrap(r)
+    return r
 
 
 W_DropWhile.typedef = TypeDef(
@@ -253,7 +253,7 @@
         self.iterable = space.iter(w_iterable)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         while True:
@@ -273,7 +273,7 @@
 def W_IFilter___new__(space, w_subtype, w_predicate, w_iterable):
     r = space.allocate_instance(W_IFilter, w_subtype)
     r.__init__(space, w_predicate, w_iterable)
-    return space.wrap(r)
+    return r
 
 W_IFilter.typedef = TypeDef(
         'itertools.ifilter',
@@ -300,7 +300,7 @@
 def W_IFilterFalse___new__(space, w_subtype, w_predicate, w_iterable):
     r = space.allocate_instance(W_IFilterFalse, w_subtype)
     r.__init__(space, w_predicate, w_iterable)
-    return space.wrap(r)
+    return r
 
 W_IFilterFalse.typedef = TypeDef(
         'itertools.ifilterfalse',
@@ -379,11 +379,11 @@
                 raise
             result = -1
         if result < minimum:
-            raise OperationError(space.w_ValueError, space.wrap(errormsg))
+            raise OperationError(space.w_ValueError, space.newtext(errormsg))
         return result
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.start >= 0:               # first call only
@@ -437,7 +437,7 @@
 def W_ISlice___new__(space, w_subtype, w_iterable, w_startstop, args_w):
     r = space.allocate_instance(W_ISlice, w_subtype)
     r.__init__(space, w_iterable, w_startstop, args_w)
-    return space.wrap(r)
+    return r
 
 W_ISlice.typedef = TypeDef(
         'itertools.islice',
@@ -465,7 +465,7 @@
         self.w_it = None
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def _advance(self):
         self.w_it = self.space.iter(self.space.next(self.w_iterables))
@@ -492,7 +492,7 @@
     r = space.allocate_instance(W_Chain, w_subtype)
     w_args = space.newtuple(args_w)
     r.__init__(space, space.iter(w_args))
-    return space.wrap(r)
+    return r
 
 def chain_from_iterable(space, w_cls, w_arg):
     """chain.from_iterable(iterable) --> chain object
@@ -501,7 +501,7 @@
     that evaluates lazily."""
     r = space.allocate_instance(W_Chain, w_cls)
     r.__init__(space, space.iter(w_arg))
-    return space.wrap(r)
+    return r
 
 W_Chain.typedef = TypeDef(
         'itertools.chain',
@@ -554,7 +554,7 @@
         self.iterators_w = iterators_w
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         # common case: 1 or 2 arguments
@@ -584,7 +584,7 @@
                     "imap() must have at least two arguments")
     r = space.allocate_instance(W_IMap, w_subtype)
     r.__init__(space, w_fun, args_w)
-    return space.wrap(r)
+    return r
 
 W_IMap.typedef = TypeDef(
         'itertools.imap',
@@ -628,7 +628,7 @@
 def W_IZip___new__(space, w_subtype, args_w):
     r = space.allocate_instance(W_IZip, w_subtype)
     r.__init__(space, space.w_None, args_w)
-    return space.wrap(r)
+    return r
 
 W_IZip.typedef = TypeDef(
         'itertools.izip',
@@ -701,7 +701,7 @@
     self.w_fillvalue = w_fillvalue
     self.active = len(self.iterators_w)
 
-    return space.wrap(self)
+    return self
 
 W_IZipLongest.typedef = TypeDef(
         'itertools.izip_longest',
@@ -726,7 +726,7 @@
         self.exhausted = False
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.exhausted:
@@ -759,7 +759,7 @@
 def W_Cycle___new__(space, w_subtype, w_iterable):
     r = space.allocate_instance(W_Cycle, w_subtype)
     r.__init__(space, w_iterable)
-    return space.wrap(r)
+    return r
 
 W_Cycle.typedef = TypeDef(
         'itertools.cycle',
@@ -790,7 +790,7 @@
         self.w_iterable = self.space.iter(w_iterable)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         w_obj = self.space.next(self.w_iterable)
@@ -799,7 +799,7 @@
 def W_StarMap___new__(space, w_subtype, w_fun, w_iterable):
     r = space.allocate_instance(W_StarMap, w_subtype)
     r.__init__(space, w_fun, w_iterable)
-    return space.wrap(r)
+    return r
 
 W_StarMap.typedef = TypeDef(
         'itertools.starmap',
@@ -855,7 +855,7 @@
     if n < 0:
         raise oefmt(space.w_ValueError, "n must be >= 0")
 
-    if space.findattr(w_iterable, space.wrap("__copy__")) is not None:
+    if space.findattr(w_iterable, space.newtext("__copy__")) is not None:
         # In this case, we don't instantiate any W_TeeIterable.
         # We just rely on doing repeated __copy__().  This case
         # includes the situation where w_iterable is already
@@ -867,8 +867,7 @@
     else:
         w_iterator = space.iter(w_iterable)
         chained_list = TeeChainedListNode()
-        iterators_w = [space.wrap(
-                           W_TeeIterable(space, w_iterator, chained_list))
+        iterators_w = [W_TeeIterable(space, w_iterator, chained_list)
                        for x in range(n)]
     return space.newtuple(iterators_w)
 
@@ -884,7 +883,7 @@
         self.chained_list = chained_list
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         chained_list = self.chained_list
@@ -899,7 +898,7 @@
     def copy_w(self):
         space = self.space
         tee_iter = W_TeeIterable(space, self.w_iterator, self.chained_list)
-        return space.wrap(tee_iter)
+        return tee_iter
 
 def W_TeeIterable___new__(space, w_subtype, w_iterable):
     # Obscure and undocumented function.  PyPy only supports w_iterable
@@ -908,8 +907,7 @@
     # semantics are then slightly different; see the XXX in lib-python's
     # test_itertools).
     myiter = space.interp_w(W_TeeIterable, w_iterable)
-    return space.wrap(W_TeeIterable(space, myiter.w_iterator,
-                                           myiter.chained_list))
+    return W_TeeIterable(space, myiter.w_iterator, myiter.chained_list)
 
 W_TeeIterable.typedef = TypeDef(
         'itertools._tee',
@@ -940,7 +938,7 @@
         self.w_key = self.space.w_None
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.exhausted:
@@ -966,7 +964,7 @@
                 self.lookahead = True
 
         self.new_group = False
-        w_iterator = self.space.wrap(W_GroupByIterator(self.space, self.index, self))
+        w_iterator = W_GroupByIterator(self.space, self.index, self)
         return self.space.newtuple([self.w_key, w_iterator])
 
     def _consume_unwanted_input(self):
@@ -1013,7 +1011,7 @@
 def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
     r = space.allocate_instance(W_GroupBy, w_subtype)
     r.__init__(space, w_iterable, w_key)
-    return space.wrap(r)
+    return r
 
 W_GroupBy.typedef = TypeDef(
         'itertools.groupby',
@@ -1048,7 +1046,7 @@
         self.exhausted = False
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.exhausted:
@@ -1076,7 +1074,7 @@
         self.w_selectors = space.iter(w_selectors)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         # No need to check for StopIteration since either w_data
@@ -1091,7 +1089,7 @@
 def W_Compress__new__(space, w_subtype, w_data, w_selectors):
     r = space.allocate_instance(W_Compress, w_subtype)
     r.__init__(space, w_data, w_selectors)
-    return space.wrap(r)
+    return r
 
 W_Compress.typedef = TypeDef(
     'itertools.compress',
@@ -1163,7 +1161,7 @@
             self.lst = None
 
     def iter_w(self, space):
-        return space.wrap(self)
+        return self
 
     def next_w(self, space):
         if self.lst is None:
@@ -1175,7 +1173,7 @@
 
 def W_Product__new__(space, w_subtype, __args__):
     arguments_w, kwds_w = __args__.unpack()
-    w_repeat = space.wrap(1)
+    w_repeat = space.newint(1)
     if kwds_w:
         if 'repeat' in kwds_w:
             w_repeat = kwds_w['repeat']
@@ -1186,7 +1184,7 @@
 
     r = space.allocate_instance(W_Product, w_subtype)
     r.__init__(space, arguments_w, w_repeat)
-    return space.wrap(r)
+    return r
 
 W_Product.typedef = TypeDef(
     'itertools.product',
@@ -1288,7 +1286,7 @@
     indices = range(len(pool_w))
     res = space.allocate_instance(W_Combinations, w_subtype)
     res.__init__(space, pool_w, indices, r)
-    return space.wrap(res)
+    return res
 
 W_Combinations.typedef = TypeDef("itertools.combinations",
     __new__ = interp2app(W_Combinations__new__),
@@ -1321,7 +1319,7 @@
     indices = [0] * r
     res = space.allocate_instance(W_CombinationsWithReplacement, w_subtype)
     res.__init__(space, pool_w, indices, r)
-    return space.wrap(res)
+    return res
 
 W_CombinationsWithReplacement.typedef = TypeDef(
     "itertools.combinations_with_replacement",
@@ -1387,7 +1385,7 @@
         r = space.gateway_nonnegint_w(w_r)
     res = space.allocate_instance(W_Permutations, w_subtype)
     res.__init__(space, pool_w, r)
-    return space.wrap(res)
+    return res
 
 W_Permutations.typedef = TypeDef("itertools.permutations",
     __new__ = interp2app(W_Permutations__new__),

From pypy.commits at gmail.com  Tue Nov  1 14:34:38 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 01 Nov 2016 11:34:38 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: warning module
Message-ID: <5818e03e.8a0d1c0a.3dce7.01cc@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88050:3e9bc4ff4247
Date: 2016-11-01 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/3e9bc4ff4247/

Log:	warning module

diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -3,14 +3,14 @@
 
 def create_filter(space, w_category, action):
     return space.newtuple([
-        space.wrap(action), space.w_None, w_category,
-        space.w_None, space.wrap(0)])
+        space.newtext(action), space.w_None, w_category,
+        space.w_None, space.newint(0)])
 
 class State:
     def __init__(self, space):
         self.init_filters(space)
         self.w_once_registry = space.newdict()
-        self.w_default_action = space.wrap("default")
+        self.w_default_action = space.newtext("default")
 
     def init_filters(self, space):
         filters_w = []
@@ -40,14 +40,14 @@
 def get_warnings_attr(space, name):
     try:
         w_module = space.getitem(space.sys.get('modules'),
-                                 space.wrap('warnings'))
+                                 space.newtext('warnings'))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
         return None
 
     try:
-        return space.getattr(w_module, space.wrap(name))
+        return space.getattr(w_module, space.newtext(name))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -83,43 +83,43 @@
 
     # setup registry
     try:
-        w_registry = space.getitem(w_globals, space.wrap("__warningregistry__"))
+        w_registry = space.getitem(w_globals, space.newtext("__warningregistry__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
         w_registry = space.newdict()
-        space.setitem(w_globals, space.wrap("__warningregistry__"), w_registry)
+        space.setitem(w_globals, space.newtext("__warningregistry__"), w_registry)
 
     # setup module
     try:
-        w_module = space.getitem(w_globals, space.wrap("__name__"))
+        w_module = space.getitem(w_globals, space.newtext("__name__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
-        w_module = space.wrap("<string>")
+        w_module = space.newtext("<string>")
 
     # setup filename
     try:
-        w_filename = space.getitem(w_globals, space.wrap("__file__"))
+        w_filename = space.getitem(w_globals, space.newtext("__file__"))
         filename = space.str_w(w_filename)
     except OperationError as e:
         if space.str_w(w_module) == '__main__':
             w_argv = space.sys.getdictvalue(space, 'argv')
             if w_argv and space.len_w(w_argv) > 0:
-                w_filename = space.getitem(w_argv, space.wrap(0))
+                w_filename = space.getitem(w_argv, space.newint(0))
                 if not space.is_true(w_filename):
-                    w_filename = space.wrap('__main__')
+                    w_filename = space.newtext('__main__')
             else:
                 # embedded interpreters don't have sys.argv
-                w_filename = space.wrap('__main__')
+                w_filename = space.newtext('__main__')
         else:
             w_filename = w_module
     else:
         lc_filename = filename.lower()
-        if (lc_filename.endswith(".pyc") or 
+        if (lc_filename.endswith(".pyc") or
             lc_filename.endswith(".pyo")):
             # strip last character
-            w_filename = space.wrap(filename[:-1])
+            w_filename = space.newtext(filename[:-1])
 
     return (w_filename, lineno, w_module, w_registry)
 
@@ -186,24 +186,24 @@
 
 def normalize_module(space, w_filename):
     if not space.is_true(w_filename):
-        return space.wrap("<unknown>")
+        return space.newtext("<unknown>")
 
     filename = space.str_w(w_filename)
     if filename.endswith(".py"):
         n = len(filename) - 3
         assert n >= 0
         filename = filename[:n]
-    return space.wrap(filename)
+    return space.newtext(filename)
 
 def show_warning(space, w_filename, lineno, w_text, w_category,
                  w_sourceline=None):
-    w_name = space.getattr(w_category, space.wrap("__name__"))
+    w_name = space.getattr(w_category, space.newtext("__name__"))
     w_stderr = space.sys.get("stderr")
 
     # Print "filename:lineno: category: text\n"
     message = "%s:%d: %s: %s\n" % (space.str_w(w_filename), lineno,
                                    space.str_w(w_name), space.str_w(w_text))
-    space.call_method(w_stderr, "write", space.wrap(message))
+    space.call_method(w_stderr, "write", space.newtext(message))
 
     # Print "  source_line\n"
     if not w_sourceline:
@@ -211,9 +211,9 @@
             # sourceline = linecache.getline(filename, lineno).strip()
             w_builtins = space.getbuiltinmodule('__builtin__')
             w_linecachemodule = space.call_method(w_builtins, '__import__',
-                                                  space.wrap("linecache"))
+                                                  space.newtext("linecache"))
             w_sourceline = space.call_method(w_linecachemodule, "getline",
-                                             w_filename, space.wrap(lineno))
+                                             w_filename, space.newint(lineno))
             w_sourceline = space.call_method(w_sourceline, "strip")
         except OperationError:
             w_sourceline = None
@@ -230,7 +230,7 @@
         if c not in ' \t\014':
             message = "  %s\n" % (line[i:],)
             break
-    space.call_method(w_stderr, "write", space.wrap(message))
+    space.call_method(w_stderr, "write", space.newtext(message))
 
 def do_warn(space, w_message, w_category, stacklevel):
     context_w = setup_context(space, stacklevel)
@@ -256,7 +256,7 @@
         w_text = w_message
         w_message = space.call_function(w_category, w_message)
 
-    w_lineno = space.wrap(lineno)
+    w_lineno = space.newint(lineno)
 
     # create key
     w_key = space.newtuple([w_text, w_category, w_lineno])
@@ -318,8 +318,8 @@
 
     # Check/get the requisite pieces needed for the loader.
     try:
-        w_loader = space.getitem(w_globals, space.wrap("__loader__"))
-        w_module_name = space.getitem(w_globals, space.wrap("__name__"))
+        w_loader = space.getitem(w_globals, space.newtext("__loader__"))
+        w_module_name = space.getitem(w_globals, space.newtext("__name__"))
     except OperationError as e:
         if not e.match(space, space.w_KeyError):
             raise
@@ -327,7 +327,7 @@
 
     # Make sure the loader implements the optional get_source() method.
     try:
-        w_get_source = space.getattr(w_loader, space.wrap("get_source"))
+        w_get_source = space.getattr(w_loader, space.newtext("get_source"))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -342,7 +342,7 @@
     w_source_list = space.call_method(w_source, "splitlines")
 
     # Get the source line.
-    w_source_line = space.getitem(w_source_list, space.wrap(lineno - 1))
+    w_source_line = space.getitem(w_source_list, space.newint(lineno - 1))
     return w_source_line
 
 @unwrap_spec(lineno=int, w_module = WrappedDefault(None),
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -213,7 +213,7 @@
 @dont_look_inside
 def get_jitcell_at_key(space, next_instr, is_being_profiled, w_pycode):
     ll_pycode = cast_instance_to_gcref(w_pycode)
-    return space.wrap(bool(jit_hooks.get_jitcell_at_key(
+    return space.newbool(bool(jit_hooks.get_jitcell_at_key(
         'pypyjit', r_uint(next_instr), int(is_being_profiled), ll_pycode)))
 
 @unwrap_spec(next_instr=int, is_being_profiled=bool, w_pycode=PyCode)
@@ -269,7 +269,7 @@
 #             return
 #         for i, op in enumerate(debug_info.operations):
 #             if op.is_guard():
-#                 w_t = space.newtuple([space.wrap(i), space.wrap(op.getopnum()), space.wrap(op.getdescr().get_jitcounter_hash())])
+#                 w_t = space.newtuple([space.newint(i), space.newint(op.getopnum()), space.newint(op.getdescr().get_jitcounter_hash())])
 #                 l_w.append(w_t)
 #         try:
 #             cache.in_recursion = True
@@ -287,7 +287,7 @@
 #             return
 #         if not space.is_true(cache.w_compile_bridge):
 #             return
-#         w_hash = space.wrap(debug_info.fail_descr.get_jitcounter_hash())
+#         w_hash = space.newint(debug_info.fail_descr.get_jitcounter_hash())
 #         try:
 #             cache.in_recursion = True
 #             try:
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -39,10 +39,10 @@
         ll_code = lltype.cast_opaque_ptr(lltype.Ptr(OBJECT),
                                          greenkey[2].getref_base())
         pycode = cast_base_ptr_to_instance(PyCode, ll_code)
-        return space.newtuple([space.wrap(pycode), space.wrap(next_instr),
+        return space.newtuple([pycode, space.newint(next_instr),
                                space.newbool(bool(is_being_profiled))])
     else:
-        return space.wrap(greenkey_repr)
+        return space.newtext(greenkey_repr)
 
 @unwrap_spec(operations=bool)
 def set_compile_hook(space, w_hook, operations=True):
@@ -152,10 +152,10 @@
         self.repr_of_resop = repr_of_resop
 
     def descr_repr(self, space):
-        return space.wrap(self.repr_of_resop)
+        return space.newtext(self.repr_of_resop)
 
     def descr_name(self, space):
-        return space.wrap(self.name)
+        return space.newtext(self.name)
 
 class GuardOp(WrappedOp):
     def __init__(self, name, offset, repr_of_resop, hash):
@@ -178,20 +178,20 @@
 
     def get_pycode(self, space):
         if self.jd_name == pypyjitdriver.name:
-            return space.getitem(self.w_greenkey, space.wrap(0))
+            return space.getitem(self.w_greenkey, space.newint(0))
         raise oefmt(space.w_AttributeError,
                     "This DebugMergePoint doesn't belong to the main Python "
                     "JitDriver")
 
     def get_bytecode_no(self, space):
         if self.jd_name == pypyjitdriver.name:
-            return space.getitem(self.w_greenkey, space.wrap(1))
+            return space.getitem(self.w_greenkey, space.newint(1))
         raise oefmt(space.w_AttributeError,
                     "This DebugMergePoint doesn't belong to the main Python "
                     "JitDriver")
 
     def get_jitdriver_name(self, space):
-        return space.wrap(self.jd_name)
+        return space.newtext(self.jd_name)
 
 WrappedOp.typedef = TypeDef(
     'ResOperation',
@@ -281,12 +281,12 @@
             code_repr = 'bridge no %d' % self.bridge_no
         else:
             code_repr = space.str_w(space.repr(self.w_green_key))
-        return space.wrap('<JitLoopInfo %s, %d operations, starting at <%s>>' %
-                          (self.jd_name, lgt, code_repr))
+        return space.newtext('<JitLoopInfo %s, %d operations, starting at <%s>>' %
+                             (self.jd_name, lgt, code_repr))
 
     def descr_get_bridge_no(self, space):
         if space.is_none(self.w_green_key):
-            return space.wrap(self.bridge_no)
+            return space.newint(self.bridge_no)
         raise oefmt(space.w_TypeError, "not a bridge")
 
 
@@ -374,15 +374,14 @@
     space.setitem_str(w_counter_times, 'TRACING', space.wrap(tr_time))
     b_time = jit_hooks.stats_get_times_value(None, Counters.BACKEND)
     space.setitem_str(w_counter_times, 'BACKEND', space.wrap(b_time))
-    return space.wrap(W_JitInfoSnapshot(space, w_times, w_counters,
-                                        w_counter_times))
+    return W_JitInfoSnapshot(space, w_times, w_counters, w_counter_times)
 
 def get_stats_asmmemmgr(space):
     """Returns the raw memory currently used by the JIT backend,
     as a pair (total_memory_allocated, memory_in_use)."""
     m1 = jit_hooks.stats_asmmemmgr_allocated(None)
     m2 = jit_hooks.stats_asmmemmgr_used(None)
-    return space.newtuple([space.wrap(m1), space.wrap(m2)])
+    return space.newtuple([space.newint(m1), space.newint(m2)])
 
 def enable_debug(space):
     """ Set the jit debugging - completely necessary for some stats to work,

From pypy.commits at gmail.com  Tue Nov  1 17:58:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 14:58:42 -0700 (PDT)
Subject: [pypy-commit] pypy openssl-1.1: close branch, ready to merge
Message-ID: <58191012.0209c20a.b73d8.fb59@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: openssl-1.1
Changeset: r88052:36d8204e4025
Date: 2016-11-01 22:55 +0100
http://bitbucket.org/pypy/pypy/changeset/36d8204e4025/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Tue Nov  1 17:58:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 01 Nov 2016 14:58:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge openssl-1.1
Message-ID: <58191014.031f1c0a.81841.eaea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88053:30e64025212d
Date: 2016-11-01 22:58 +0100
http://bitbucket.org/pypy/pypy/changeset/30e64025212d/

Log:	hg merge openssl-1.1

	Support for OpenSSL version 1.1 (in addition to version 1.0). Thanks
	tumbleweed for doing this! Tested on Linux (1.1, 1.0) and on Windows
	and Mac (1.0 only).

	Tumbleweed says LibreSSL still segfaults _ssl tests and fails
	_hashlib tests, just like previously.

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names
+from pypy.module._hashlib.interp_hashlib import (
+    algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC)
 
 
 class Module(MixedModule):
@@ -13,6 +14,9 @@
     for name in algorithms:
         interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
 
+    if HAS_FAST_PKCS5_PBKDF2_HMAC:
+        interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac'
+
     def startup(self, space):
         w_meth_names = fetch_names(space)
         space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -8,7 +8,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.thread.os_lock import Lock
 
@@ -58,23 +58,26 @@
     def __init__(self, space, name, copy_from=NULL_CTX):
         self.name = name
         digest_type = self.digest_type_by_name(space)
-        self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
+        self.digest_size = ropenssl.EVP_MD_size(digest_type)
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
-        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
         rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
         try:
             if copy_from:
-                ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
+                    raise ValueError
             else:
                 ropenssl.EVP_DigestInit(ctx, digest_type)
             self.ctx = ctx
         except:
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
             raise
         self.register_finalizer(space)
 
@@ -82,8 +85,7 @@
         ctx = self.ctx
         if ctx:
             self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-            ropenssl.EVP_MD_CTX_cleanup(ctx)
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
 
     def digest_type_by_name(self, space):
         digest_type = ropenssl.EVP_get_digestbyname(self.name)
@@ -128,21 +130,26 @@
 
     def get_block_size(self, space):
         digest_type = self.digest_type_by_name(space)
-        block_size = rffi.getintfield(digest_type, 'c_block_size')
+        block_size = ropenssl.EVP_MD_block_size(digest_type)
         return space.wrap(block_size)
 
     def get_name(self, space):
         return space.wrap(self.name)
 
     def _digest(self, space):
-        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
+        try:
             with self.lock:
-                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx):
+                    raise ValueError
             digest_size = self.digest_size
             with rffi.scoped_alloc_buffer(digest_size) as buf:
                 ropenssl.EVP_DigestFinal(ctx, buf.raw, None)
-                ropenssl.EVP_MD_CTX_cleanup(ctx)
                 return buf.str(digest_size)
+        finally:
+            ropenssl.EVP_MD_CTX_free(ctx)
 
 
 W_Hash.typedef = TypeDef(
@@ -177,3 +184,27 @@
 for _name in algorithms:
     _newname = 'new_%s' % (_name,)
     globals()[_newname] = make_new_hash(_name, _newname)
+
+
+HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
+if HAS_FAST_PKCS5_PBKDF2_HMAC:
+    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+                 w_dklen=WrappedDefault(None))
+    def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
+        digest = ropenssl.EVP_get_digestbyname(name)
+        if not digest:
+            raise oefmt(space.w_ValueError, "unknown hash function")
+        if space.is_w(w_dklen, space.w_None):
+            dklen = ropenssl.EVP_MD_size(digest)
+        else:
+            dklen = space.int_w(w_dklen)
+        if dklen < 1:
+            raise oefmt(space.w_ValueError,
+                        "key length must be greater than 0.")
+        with rffi.scoped_alloc_buffer(dklen) as buf:
+            r = ropenssl.PKCS5_PBKDF2_HMAC(
+                password, len(password), salt, len(salt), rounds, digest,
+                dklen, buf.raw)
+            if not r:
+                raise ValueError
+            return space.wrap(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -15,17 +15,19 @@
 
     def test_attributes(self):
         import hashlib
-        for name, expected_size in {'md5': 16,
-                                    'sha1': 20,
-                                    'sha224': 28,
-                                    'sha256': 32,
-                                    'sha384': 48,
-                                    'sha512': 64,
-                                    }.items():
+        for name, (expected_size, expected_block_size) in {
+                'md5': (16, 64),
+                'sha1': (20, 64),
+                'sha224': (28, 64),
+                'sha256': (32, 64),
+                'sha384': (48, 128),
+                'sha512': (64, 128),
+                }.items():
             h = hashlib.new(name)
             assert h.name == name
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -46,6 +48,7 @@
             h = py_new(name)('')
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -108,3 +111,13 @@
             got.decode('hex')
             if expected is not None:
                 assert got == expected
+
+    def test_pbkdf2(self):
+        try:
+            from _hashlib import pbkdf2_hmac
+        except ImportError:
+            skip("Requires OpenSSL >= 1.1")
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -33,9 +33,10 @@
 PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
 
 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
+ PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
  PY_SSL_VERSION_TLS1_2) = range(6)
 
+
 SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
 SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
@@ -68,11 +69,12 @@
 constants["HAS_NPN"] = HAS_NPN
 constants["HAS_ALPN"] = HAS_ALPN
 
+constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
+constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 if not OPENSSL_NO_SSL3:
     constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 if HAVE_TLSv1_2:
     constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
@@ -111,7 +113,10 @@
         err_reason = libssl_ERR_GET_REASON(errcode)
         reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
         lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        msg = rffi.charp2str(libssl_ERR_reason_error_string(errcode))
+        raw_msg = libssl_ERR_reason_error_string(errcode)
+        msg = None
+        if raw_msg:
+            msg = rffi.charp2str(raw_msg)
     if not msg:
         msg = "unknown error"
     if reason_str and lib_str:
@@ -637,9 +642,12 @@
         if not self.ssl:
             return space.w_None
         comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method or intmask(comp_method[0].c_type) == NID_undef:
+        if not comp_method:
             return space.w_None
-        short_name = libssl_OBJ_nid2sn(comp_method[0].c_type)
+        method_type = intmask(libssl_COMP_get_type(comp_method))
+        if method_type == NID_undef:
+            return space.w_None
+        short_name = libssl_COMP_get_name(comp_method)
         if not short_name:
             return space.w_None
         return space.wrap(rffi.charp2str(short_name))
@@ -795,7 +803,7 @@
     for index in range(entry_count):
         entry = libssl_X509_NAME_get_entry(xname, index)
         # check to see if we've gotten to a new RDN
-        entry_level = intmask(entry[0].c_set)
+        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
         if rdn_level >= 0:
             if rdn_level != entry_level:
                 # yes, new RDN
@@ -846,8 +854,9 @@
                                 "No method for internalizing subjectAltName!'")
 
             with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                p_ptr[0] = ext[0].c_value.c_data
-                length = intmask(ext[0].c_value.c_length)
+                ext_value = libssl_X509_EXTENSION_get_data(ext)
+                p_ptr[0] = ext_value.c_data
+                length = intmask(ext_value.c_length)
                 null = lltype.nullptr(rffi.VOIDP.TO)
                 if method[0].c_it:
                     names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
@@ -967,10 +976,8 @@
     if OPENSSL_VERSION_NUMBER >= 0x10001000:
         # Calls x509v3_cache_extensions and sets up crldp
         libssl_X509_check_ca(certificate)
-        dps = certificate[0].c_crldp
-    else:
-        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-            certificate, NID_crl_distribution_points, None, None))
+    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+        certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
@@ -1268,14 +1275,14 @@
     @staticmethod
     @unwrap_spec(protocol=int)
     def descr_new(space, w_subtype, protocol):
-        if protocol == PY_SSL_VERSION_TLS1:
+        if protocol == PY_SSL_VERSION_TLS:
+            method = libssl_TLS_method()
+        elif protocol == PY_SSL_VERSION_TLS1:
             method = libssl_TLSv1_method()
         elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
             method = libssl_SSLv3_method()
         elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
             method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_SSL23:
-            method = libssl_SSLv23_method()
         elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
             method = libssl_TLSv1_1_method()
         elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
@@ -1303,6 +1310,7 @@
             # OpenSSL 1.0.2+), or use prime256v1 by default.
             # This is Apache mod_ssl's initialization
             # policy, so we should be safe.
+            # OpenSSL 1.1 has it enabled by default.
             if libssl_SSL_CTX_set_ecdh_auto:
                 libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
@@ -1390,20 +1398,22 @@
 
     def descr_get_verify_flags(self, space):
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         return space.wrap(flags)
 
     def descr_set_verify_flags(self, space, w_obj):
         new_flags = space.int_w(w_obj)
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         flags_clear = flags & ~new_flags
         flags_set = ~flags & new_flags
         if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                store[0].c_param, flags_clear):
+                param, flags_clear):
             raise _ssl_seterror(space, None, 0)
         if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                store[0].c_param, flags_set):
+                param, flags_set):
             raise _ssl_seterror(space, None, 0)
 
     def descr_get_check_hostname(self, space):
@@ -1614,14 +1624,16 @@
         x509 = 0
         x509_ca = 0
         crl = 0
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) == X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
+            if obj_type == X509_LU_X509:
                 x509 += 1
                 if libssl_X509_check_ca(
-                        libssl_pypy_X509_OBJECT_data_x509(obj)):
+                        libssl_X509_OBJECT_get0_X509(obj)):
                     x509_ca += 1
-            elif intmask(obj.c_type) == X509_LU_CRL:
+            elif obj_type == X509_LU_CRL:
                 crl += 1
             else:
                 # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
@@ -1660,13 +1672,14 @@
             binary_mode = False
         rlist = []
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) != X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
                 # not a x509 cert
                 continue
             # CA for any purpose
-            cert = libssl_pypy_X509_OBJECT_data_x509(obj)
+            cert = libssl_X509_OBJECT_get0_X509(obj)
             if not libssl_X509_check_ca(cert):
                 continue
             if binary_mode:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -77,7 +77,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
             ss = ctx._wrap_socket(s, 0)
             assert ss.context is ctx
         exc = raises(_socket.error, ss.do_handshake)
@@ -95,7 +95,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0)
+            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLS)._wrap_socket(s, 0)
         s.close()
         exc = raises(_ssl.SSLError, ss.write, "data")
         assert exc.value.message == 'Underlying socket has been closed.'
@@ -123,13 +123,13 @@
 
     def test_context(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         raises(ValueError, _ssl._SSLContext, -1)
 
         assert type(s.options) is long
-        assert s.options & _ssl.OP_NO_SSLv2
-        s.options &= ~_ssl.OP_NO_SSLv2
-        assert not s.options & _ssl.OP_NO_SSLv2
+        assert s.options & _ssl.OP_NO_SSLv3
+        s.options &= ~_ssl.OP_NO_SSLv3
+        assert not s.options & _ssl.OP_NO_SSLv3
         raises(TypeError, "s.options = 2.5")
 
         assert not s.check_hostname
@@ -159,7 +159,7 @@
 
     def test_set_default_verify_paths(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s.set_default_verify_paths()
 
 
@@ -253,13 +253,44 @@
         if not _ssl.HAS_NPN:
             skip("NPN requires OpenSSL 1.0.1 or greater")
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s._sock, True,
                               server_hostname="svn.python.org")
         self.s.close()
         del ss; gc.collect()
 
+    def test_peer_certificate(self):
+        import _ssl, gc
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ss = ctx._wrap_socket(self.s._sock, False)
+        ss.do_handshake()
+        assert isinstance(ss.peer_certificate(der=True), bytes)
+        assert isinstance(ss.peer_certificate(), dict)
+        self.s.close()
+        del ss; gc.collect()
+
+    def test_peer_certificate_verify(self):
+        import _ssl, ssl, gc
+        paths = ssl.get_default_verify_paths()
+        if not paths.capath and not paths.cafile:
+            skip("ssl.get_default_verify_paths() failed to return any path")
+
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.verify_mode = _ssl.CERT_REQUIRED
+        ctx.load_verify_locations(capath=paths.capath, cafile=paths.cafile)
+
+        ss = ctx._wrap_socket(self.s._sock, False)
+        try:
+            ss.do_handshake()
+        except _ssl.SSLError as e:
+            if e.reason == 'CERTIFICATE_VERIFY_FAILED':
+                skip("Certificate verification failed. "
+                     "Most likely we just don't have any CA certificates.")
+        assert ss.peer_certificate()
+        self.s.close()
+        del ss; gc.collect()
+
     def test_tls_unique_cb(self):
         import ssl, sys, gc
         ss = ssl.wrap_socket(self.s)
@@ -325,7 +356,7 @@
 
     def test_load_cert_chain(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
         exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
@@ -344,11 +375,11 @@
 
     def test_load_verify_locations(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         ctx.load_verify_locations(cafile=self.keycert, capath=None)
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         with open(self.keycert) as f:
             cacert_pem = f.read().decode('ascii')
         ctx.load_verify_locations(cadata=cacert_pem)
@@ -356,7 +387,7 @@
 
     def test_get_ca_certs(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         assert ctx.get_ca_certs() == []
         ctx.load_verify_locations(self.python_org_cert)
@@ -370,7 +401,7 @@
 
     def test_cert_store_stats(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
         ctx.load_cert_chain(self.keycert)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
@@ -379,7 +410,7 @@
 
     def test_load_dh_params(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
@@ -389,7 +420,7 @@
 
     def test_set_ecdh_curve(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.set_ecdh_curve("prime256v1")
         raises(ValueError, ctx.set_ecdh_curve, "foo")
 
@@ -434,7 +465,7 @@
     def test_lib_reason(self):
         # Test the library and reason attributes
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         exc = raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
         assert exc.value.library == 'PEM'
         assert exc.value.reason == 'NO_START_LINE'
@@ -445,7 +476,7 @@
         # Check that the appropriate SSLError subclass is raised
         # (this only tests one of them)
         import _ssl, _socket
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s = _socket.socket()
         try:
             s.bind(("127.0.0.1", 0))
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -57,8 +57,19 @@
         '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)',
         '#define pypy_GENERAL_NAME_uri(name) (name->d.uniformResourceIdentifier)',
         '#define pypy_GENERAL_NAME_pop_free(names) (sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free))',
-        '#define pypy_X509_OBJECT_data_x509(obj) (obj->data.x509)',
         '#define pypy_DIST_POINT_fullname(obj) (obj->distpoint->name.fullname)',
+        # Backwards compatibility for functions introduced in 1.1
+        '#if (OPENSSL_VERSION_NUMBER < 0x10100000) || defined(LIBRESSL_VERSION_NUMBER)\n'
+        '#  define COMP_get_name(meth) (meth->name)\n'
+        '#  define COMP_get_type(meth) (meth->type)\n'
+        '#  define X509_NAME_ENTRY_set(ne) (ne->set)\n'
+        '#  define X509_OBJECT_get0_X509(obj) (obj->data.x509)\n'
+        '#  define X509_OBJECT_get_type(obj) (obj->type)\n'
+        '#  define X509_STORE_get0_objects(store) (store->objs)\n'
+        '#  define X509_STORE_get0_param(store) (store->param)\n'
+        '#else\n'
+        '#  define OPENSSL_NO_SSL2\n'
+        '#endif',
     ],
 )
 
@@ -88,6 +99,7 @@
             "OPENSSL_EXPORT_VAR_AS_FUNCTION")
     OPENSSL_VERSION_NUMBER = rffi_platform.ConstantInteger(
         "OPENSSL_VERSION_NUMBER")
+    LIBRESSL = rffi_platform.Defined("LIBRESSL_VERSION_NUMBER")
 
 cconfig = rffi_platform.configure(CConfigBootstrap)
 if cconfig["OPENSSL_EXPORT_VAR_AS_FUNCTION"]:
@@ -95,23 +107,10 @@
 else:
     ASN1_ITEM_EXP = ASN1_ITEM
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
+LIBRESSL = cconfig["LIBRESSL"]
+OPENSSL_1_1 = OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
-     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
-    eci.pre_include_bits = ()
-    eci.post_include_bits = ()
-    raise Exception("""OpenSSL version >= 1.1 not supported yet.
-
-    This program requires OpenSSL version 1.0.x, and may also
-    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
-    some work to update to; contributions are welcome.  Sorry,
-    you need to install an older version of OpenSSL for now.
-    Make sure this older version is the one picked up by this
-    program when it runs the compiler.
-    
-    This is the configuration used: %r""" % (eci,))
-
 
 class CConfig:
     _compilation_info_ = eci
@@ -210,28 +209,10 @@
     OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger(
         "OBJ_NAME_TYPE_MD_METH")
 
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        X509_st = rffi_platform.Struct(
-            'struct x509_st',
-            [('crldp', stack_st_DIST_POINT)])
-
     # Some structures, with only the fields used in the _ssl module
-    X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st',
-                                              [('set', rffi.INT)])
     asn1_string_st = rffi_platform.Struct('struct asn1_string_st',
                                           [('length', rffi.INT),
                                            ('data', rffi.CCHARP)])
-    X509_extension_st = rffi_platform.Struct(
-        'struct X509_extension_st',
-        [('value', ASN1_STRING)])
-    x509_store_st = rffi_platform.Struct(
-        'struct x509_store_st',
-        [('objs', stack_st_X509_OBJECT),
-         ('param', X509_VERIFY_PARAM)])
-    x509_object_st = rffi_platform.Struct(
-        'struct x509_object_st',
-        [('type', rffi.INT)])
-
     X509_LU_X509 = rffi_platform.ConstantInteger("X509_LU_X509")
     X509_LU_CRL = rffi_platform.ConstantInteger("X509_LU_CRL")
 
@@ -244,12 +225,6 @@
     GENERAL_NAME_st = rffi_platform.Struct(
         'struct GENERAL_NAME_st',
         [('type', rffi.INT)])
-    EVP_MD_st = rffi_platform.Struct(
-        'EVP_MD',
-        [('md_size', rffi.INT),
-         ('block_size', rffi.INT)])
-    EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
-    EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
     OBJ_NAME_st = rffi_platform.Struct(
         'OBJ_NAME',
@@ -257,10 +232,6 @@
          ('name', rffi.CCHARP),
          ])
 
-    COMP_METHOD_st = rffi_platform.Struct(
-        'struct comp_method_st',
-        [('type', rffi.INT),])
-
     ACCESS_DESCRIPTION_st = rffi_platform.Struct(
         'struct ACCESS_DESCRIPTION_st',
         [('method', ASN1_OBJECT),
@@ -275,14 +246,12 @@
 SSL_CIPHER = rffi.COpaquePtr('SSL_CIPHER')
 SSL = rffi.COpaquePtr('SSL')
 BIO = rffi.COpaquePtr('BIO')
-if OPENSSL_VERSION_NUMBER >= 0x10001000:
-    X509 = rffi.CArrayPtr(X509_st)
-else:
-    X509 = rffi.COpaquePtr('X509')
-X509_NAME_ENTRY = rffi.CArrayPtr(X509_name_entry_st)
-X509_EXTENSION = rffi.CArrayPtr(X509_extension_st)
-X509_STORE = rffi.CArrayPtr(x509_store_st)
-X509_OBJECT = lltype.Ptr(x509_object_st)
+X509 = rffi.COpaquePtr('X509')
+X509_OBJECT = rffi.COpaquePtr('X509_OBJECT')
+COMP_METHOD = rffi.COpaquePtr('COMP_METHOD')
+X509_NAME_ENTRY = rffi.COpaquePtr('X509_NAME_ENTRY')
+X509_EXTENSION = rffi.COpaquePtr('X509_EXTENSION')
+X509_STORE = rffi.COpaquePtr('X509_STORE')
 X509V3_EXT_METHOD = rffi.CArrayPtr(v3_ext_method)
 ASN1_STRING.TO.become(asn1_string_st)
 ASN1_TIME = rffi.COpaquePtr('ASN1_TIME')
@@ -290,9 +259,9 @@
 GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES')
 GENERAL_NAME.TO.become(GENERAL_NAME_st)
 OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
-COMP_METHOD = rffi.CArrayPtr(COMP_METHOD_st)
 ACCESS_DESCRIPTION = rffi.CArrayPtr(ACCESS_DESCRIPTION_st)
 
+
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_OPENSSL_FINISHED = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df and \
@@ -314,8 +283,10 @@
     globals()['libssl_' + name] = external(
         name, argtypes, restype, **kw)
 
-ssl_external('SSL_load_error_strings', [], lltype.Void)
-ssl_external('SSL_library_init', [], rffi.INT)
+ssl_external('SSL_load_error_strings', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
+ssl_external('SSL_library_init', [], rffi.INT,
+    macro=OPENSSL_1_1 or None)
 ssl_external('CRYPTO_num_locks', [], rffi.INT)
 ssl_external('CRYPTO_set_locking_callback',
              [lltype.Ptr(lltype.FuncType(
@@ -341,7 +312,11 @@
     ssl_external('TLSv1_2_method', [], SSL_METHOD)
 ssl_external('SSLv2_method', [], SSL_METHOD)
 ssl_external('SSLv3_method', [], SSL_METHOD)
-ssl_external('SSLv23_method', [], SSL_METHOD)
+# Windows note: fails in untranslated tests if the following function is
+# made 'macro=True'.  Not sure I want to dig into the reason for that mess.
+libssl_TLS_method = external(
+    'TLS_method' if OPENSSL_1_1 else 'SSLv23_method',
+    [], SSL_METHOD)
 ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT,
@@ -370,7 +345,7 @@
              lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tmp_ecdh', [SSL_CTX, EC_KEY], lltype.Void, macro=True)
-if OPENSSL_VERSION_NUMBER >= 0x10002000:
+if OPENSSL_VERSION_NUMBER >= 0x10002000 and not OPENSSL_1_1:
     ssl_external('SSL_CTX_set_ecdh_auto', [SSL_CTX, rffi.INT], lltype.Void,
                  macro=True)
 else:
@@ -416,6 +391,8 @@
 ssl_external('X509_NAME_get_entry', [X509_NAME, rffi.INT], X509_NAME_ENTRY)
 ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT)
 ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING)
+ssl_external('X509_NAME_ENTRY_set', [X509_NAME_ENTRY], rffi.INT,
+    macro=(not OPENSSL_1_1) or None)
 ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT, save_err=SAVE_ERR)
 ssl_external('X509_free', [X509], lltype.Void, releasegil=False)
 ssl_external('X509_check_ca', [X509], rffi.INT)
@@ -427,11 +404,16 @@
 ssl_external('X509_get_ext', [X509, rffi.INT], X509_EXTENSION)
 ssl_external('X509_get_ext_d2i', [X509, rffi.INT, rffi.VOIDP, rffi.VOIDP], rffi.VOIDP)
 ssl_external('X509V3_EXT_get', [X509_EXTENSION], X509V3_EXT_METHOD)
+ssl_external('X509_EXTENSION_get_data', [X509_EXTENSION], ASN1_STRING)
 
 ssl_external('X509_VERIFY_PARAM_get_flags', [X509_VERIFY_PARAM], rffi.ULONG)
 ssl_external('X509_VERIFY_PARAM_set_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_VERIFY_PARAM_clear_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_STORE_add_cert', [X509_STORE, X509], rffi.INT)
+ssl_external('X509_STORE_get0_objects', [X509_STORE], stack_st_X509_OBJECT,
+    macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_STORE_get0_param', [X509_STORE], X509_VERIFY_PARAM,
+    macro=bool(not OPENSSL_1_1) or None)
 
 ssl_external('X509_get_default_cert_file_env', [], rffi.CCHARP)
 ssl_external('X509_get_default_cert_file', [], rffi.CCHARP)
@@ -468,8 +450,14 @@
              macro=True)
 ssl_external('sk_X509_OBJECT_value', [stack_st_X509_OBJECT, rffi.INT],
              X509_OBJECT, macro=True)
-ssl_external('pypy_X509_OBJECT_data_x509', [X509_OBJECT], X509,
-             macro=True)
+ssl_external('X509_OBJECT_get0_X509', [X509_OBJECT], X509,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_OBJECT_get_type', [X509_OBJECT], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_name', [COMP_METHOD], rffi.CCHARP,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_type', [COMP_METHOD], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('sk_DIST_POINT_num', [stack_st_DIST_POINT], rffi.INT,
              macro=True)
 ssl_external('sk_DIST_POINT_value', [stack_st_DIST_POINT, rffi.INT], DIST_POINT,
@@ -511,8 +499,12 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
-libssl_OPENSSL_free = libssl_CRYPTO_free
+if OPENSSL_1_1:
+    ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
+else:
+    ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
+    libssl_OPENSSL_free = libssl_CRYPTO_free
+    del libssl_CRYPTO_free
 
 ssl_external('SSL_write', [SSL, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=SAVE_ERR)
@@ -576,10 +568,11 @@
             SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = lltype.Ptr(EVP_MD_st)
+EVP_MD = rffi.COpaquePtr('EVP_MD')
 
 OpenSSL_add_all_digests = external(
-    'OpenSSL_add_all_digests', [], lltype.Void)
+    'OpenSSL_add_all_digests', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
 EVP_get_digestbyname = external(
     'EVP_get_digestbyname',
     [rffi.CCHARP], EVP_MD)
@@ -592,10 +585,25 @@
 EVP_DigestFinal = external(
     'EVP_DigestFinal',
     [EVP_MD_CTX, rffi.CCHARP, rffi.VOIDP], rffi.INT)
+EVP_MD_size = external(
+    'EVP_MD_size', [EVP_MD], rffi.INT)
+EVP_MD_block_size = external(
+    'EVP_MD_block_size', [EVP_MD], rffi.INT)
 EVP_MD_CTX_copy = external(
     'EVP_MD_CTX_copy', [EVP_MD_CTX, EVP_MD_CTX], rffi.INT)
-EVP_MD_CTX_cleanup = external(
-    'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, releasegil=False)
+EVP_MD_CTX_new = external(
+    'EVP_MD_CTX_new' if OPENSSL_1_1 else 'EVP_MD_CTX_create',
+    [], EVP_MD_CTX)
+EVP_MD_CTX_free = external(
+    'EVP_MD_CTX_free' if OPENSSL_1_1 else 'EVP_MD_CTX_destroy',
+    [EVP_MD_CTX], lltype.Void, releasegil=False)
+
+if OPENSSL_1_1:
+    PKCS5_PBKDF2_HMAC = external('PKCS5_PBKDF2_HMAC', [
+        rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT, rffi.INT, EVP_MD,
+        rffi.INT, rffi.CCHARP], rffi.INT)
+else:
+    PKCS5_PBKDF2_HMAC = None
 
 OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType(
         [OBJ_NAME, rffi.VOIDP], lltype.Void))
@@ -606,7 +614,7 @@
 # Used for adding memory pressure. Last number is an (under?)estimate of
 # EVP_PKEY_CTX's size.
 # XXX: Make a better estimate here
-HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \
+HASH_MALLOC_SIZE = 120 + 48 \
         + rffi.sizeof(EVP_MD) * 2 + 208
 
 def init_ssl():
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1575,15 +1575,13 @@
 
         class A:
             def __init__(self):
-                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
-                    flavor='raw')
                 digest = ropenssl.EVP_get_digestbyname('sha1')
+                self.ctx = ropenssl.EVP_MD_CTX_new()
                 ropenssl.EVP_DigestInit(self.ctx, digest)
                 rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
 
             def __del__(self):
-                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
-                lltype.free(self.ctx, flavor='raw')
+                ropenssl.EVP_MD_CTX_free(self.ctx)
         #A() --- can't call it here?? get glibc crashes on tannit64
         def f():
             am1 = am2 = am3 = None

From pypy.commits at gmail.com  Tue Nov  1 18:35:55 2016
From: pypy.commits at gmail.com (stefanor)
Date: Tue, 01 Nov 2016 15:35:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix minor transcription error from
 cpython C extension
Message-ID: <581918cb.42151c0a.5f2b.cfd9@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r88054:ed8820ed0ae8
Date: 2016-11-01 15:35 -0700
http://bitbucket.org/pypy/pypy/changeset/ed8820ed0ae8/

Log:	Fix minor transcription error from cpython C extension

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -524,8 +524,8 @@
             maxindex = XML_GetSpecifiedAttributeCount(self.itself)
         else:
             maxindex = 0
-        while attrs[maxindex]:
-            maxindex += 2 # copied
+            while attrs[maxindex]:
+                maxindex += 2 # copied
 
         if self.ordered_attributes:
             w_attrs = space.newlist([

From pypy.commits at gmail.com  Tue Nov  1 19:05:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 01 Nov 2016 16:05:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: document recent changes for upcoming
 release
Message-ID: <58191fa8.0370c20a.4405c.182c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88055:b8196f0b4a7b
Date: 2016-11-02 01:04 +0200
http://bitbucket.org/pypy/pypy/changeset/b8196f0b4a7b/

Log:	document recent changes for upcoming release

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -16,6 +16,9 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
+We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+well as maintaining support for previous versions.
+
 XXX
 
 As always, this release fixed many issues and bugs raised by the
@@ -72,7 +75,7 @@
 * New features
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
-  * Backport fixes to rposix on windows from py2.5
+  * Backport fixes to rposix on windows from py3.5
   * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
   * Add ``inode`` to ``scandir()`` on posix systems
   * Support more attributes on ``super``
@@ -98,6 +101,8 @@
   * Support translation on FreeBSD running on PowerPC
   * Implement ``__rmod__`` on ``str`` and ``unicode`` types
   * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors

From pypy.commits at gmail.com  Wed Nov  2 04:05:28 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 02 Nov 2016 01:05:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Pickle protocol 4: Call
 __getnewargs_ex__(), implemented when __new__() requires keyword arguments.
Message-ID: <58199e48.120f1c0a.748ba.6b8b@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88056:82efc04d5dda
Date: 2016-11-02 09:04 +0100
http://bitbucket.org/pypy/pypy/changeset/82efc04d5dda/

Log:	Pickle protocol 4: Call __getnewargs_ex__(), implemented when
	__new__() requires keyword arguments.

diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -18,17 +18,27 @@
     import copyreg
     return copyreg._reduce_ex(obj, proto)
 
-def reduce_2(obj):
-    cls = obj.__class__
+def _getnewargs(obj):
 
     try:
-        getnewargs = obj.__getnewargs__
+        getnewargs = obj.__getnewargs_ex__
     except AttributeError:
-        args = ()
+        try:
+            getnewargs = obj.__getnewargs__
+        except AttributeError:
+            args = ()
+        else:
+            args = getnewargs()
+        kwargs = None
     else:
-        args = getnewargs()
-        if not isinstance(args, tuple):
-            raise TypeError("__getnewargs__ should return a tuple")
+        args, kwargs = getnewargs()
+
+    if not isinstance(args, tuple):
+        raise TypeError("__getnewargs__ should return a tuple")
+    return args, kwargs
+
+def _getstate(obj):
+    cls = obj.__class__
 
     try:
         getstate = obj.__getstate__
@@ -48,15 +58,32 @@
                 state = state, slots
     else:
         state = getstate()
+    return state
 
+def reduce_2(obj, proto):
+    cls = obj.__class__
+
+    import copyreg
+
+    args, kwargs = _getnewargs(obj)
+
+    if not kwargs:
+       newobj = copyreg.__newobj__
+       args2 = (cls,) + args
+    elif proto >= 4:
+       newobj = copyreg.__newobj_ex__
+       args2 = (cls, args, kwargs)
+    else:
+       raise ValueError("must use protocol 4 or greater to copy this "
+                        "object; since __getnewargs_ex__ returned "
+                        "keyword arguments.")
+
+    state = _getstate(obj)
     listitems = iter(obj) if isinstance(obj, list) else None
     dictitems = iter(obj.items()) if isinstance(obj, dict) else None
 
-    import copyreg
-    newobj = copyreg.__newobj__
+    return newobj, args2, state, listitems, dictitems
 
-    args2 = (cls,) + args
-    return newobj, args2, state, listitems, dictitems
 
 def slotnames(cls):
     if not isinstance(cls, type):
@@ -169,9 +196,9 @@
 
 @unwrap_spec(proto=int)
 def descr__reduce__(space, w_obj, proto=0):
+    w_proto = space.wrap(proto)
     if proto >= 2:
-        return reduce_2(space, w_obj)
-    w_proto = space.wrap(proto)
+        return reduce_2(space, w_obj, w_proto)
     return reduce_1(space, w_obj, w_proto)
 
 @unwrap_spec(proto=int)
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -59,6 +59,22 @@
         s = X().__reduce__()
         assert s[-1] == ':-)'
 
+    def test_getnewargs_ex(self):
+        class NamedInt(int):
+            def __new__(cls, name, **kwargs):
+                if len(kwargs) == 0:
+                    raise TypeError("name and value must be specified")
+                self = int.__new__(cls, kwargs['value'])
+                self._name = name
+                return self
+            def __getnewargs_ex__(self):
+                return (self._name,), dict(value=int(self))
+        import copyreg
+        assert NamedInt("Name", value=42).__reduce__(4) == (
+            copyreg.__newobj_ex__,
+            (NamedInt, ('Name',), dict(value=42)),
+            dict(_name='Name'), None, None)
+
     def test_default_format(self):
         class x(object):
             def __str__(self):

From pypy.commits at gmail.com  Wed Nov  2 04:32:53 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 02 Nov 2016 01:32:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix "namereplace" handler for unknown
 characters.
Message-ID: <5819a4b5.87dc1c0a.c38a5.042e@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88057:08fdd579df29
Date: 2016-11-02 09:31 +0100
http://bitbucket.org/pypy/pypy/changeset/08fdd579df29/

Log:	Fix "namereplace" handler for unknown characters. Also convert other
	handlers to return a bytes string, python3 allows this now (no need
	to encode the replacement string again)

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,7 +1,8 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
-from rpython.rlib.runicode import code_to_unichr, MAXUNICODE
+from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.runicode import (
+    code_to_unichr, MAXUNICODE, raw_unicode_escape_helper)
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -245,7 +246,7 @@
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
         end = space.int_w(w_end)
-        builder = UnicodeBuilder()
+        builder = StringBuilder()
         pos = start
         while pos < end:
             code = ord(obj[pos])
@@ -255,9 +256,9 @@
                 code |= ord(obj[pos+1]) & 0x03FF
                 code += 0x10000
                 pos += 1
-            builder.append(u"&#")
-            builder.append(unicode(str(code)))
-            builder.append(u";")
+            builder.append("&#")
+            builder.append(str(code))
+            builder.append(";")
             pos += 1
         return space.newtuple([space.wrap(builder.build()), w_end])
     else:
@@ -271,25 +272,11 @@
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
         end = space.int_w(w_end)
-        builder = UnicodeBuilder()
+        builder = StringBuilder()
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            num = hex(oc)
-            if (oc >= 0x10000):
-                builder.append(u"\\U")
-                zeros = 8
-            elif (oc >= 0x100):
-                builder.append(u"\\u")
-                zeros = 4
-            else:
-                builder.append(u"\\x")
-                zeros = 2
-            lnum = len(num)
-            nb = zeros + 2 - lnum # num starts with '0x'
-            if nb > 0:
-                builder.append_multiple_char(u'0', nb)
-            builder.append_slice(unicode(num), 2, lnum)
+            raw_unicode_escape_helper(builder, oc)
             pos += 1
         return space.newtuple([space.wrap(builder.build()), w_end])
     else:
@@ -307,9 +294,14 @@
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            builder.append('\\N{')
-            builder.append(unicodedb.name(oc))
-            builder.append('}')
+            try:
+                name = unicodedb.name(oc)
+            except KeyError:
+                raw_unicode_escape_helper(builder, oc)
+            else:
+                builder.append('\\N{')
+                builder.append(name)
+                builder.append('}')
             pos += 1
         return space.newtuple([space.newbytes(builder.build()), w_end])
     else:
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -568,6 +568,7 @@
         assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'namereplace') == (
             b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}'
             b'\\N{CJK UNIFIED IDEOGRAPH-8000}')
+        assert '[\uDC80]'.encode('utf-8', 'namereplace') == b'[\\udc80]'
 
     def test_surrogateescape(self):
         assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b'

From pypy.commits at gmail.com  Wed Nov  2 05:30:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Nov 2016 02:30:28 -0700 (PDT)
Subject: [pypy-commit] cffi default: Bump version number to 1.9.0
Message-ID: <5819b234.0777c20a.532be.7681@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2804:649876993d17
Date: 2016-11-02 10:30 +0100
http://bitbucket.org/cffi/cffi/changeset/649876993d17/

Log:	Bump version number to 1.9.0

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.8.4"
+#define CFFI_VERSION  "1.9.0"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.8'
+version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.4'
+release = '1.9.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,7 +51,7 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.8.4.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.0.tar.gz
 
    - MD5: ...
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -153,7 +153,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.8.4',
+        version='1.9.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Wed Nov  2 05:40:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Nov 2016 02:40:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/649876993d17: bump
 version number to 1.9.0
Message-ID: <5819b49c.28c9c20a.cb9c0.7949@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88058:bc1ab6e3f408
Date: 2016-11-02 10:39 +0100
http://bitbucket.org/pypy/pypy/changeset/bc1ab6e3f408/

Log:	update to cffi/649876993d17: bump version number to 1.9.0

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.4"
+VERSION = "1.9.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Wed Nov  2 06:33:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 03:33:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-ssl: add the renamed cffi build script
Message-ID: <5819c113.c1d81c0a.f3776.ad28@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88059:1ee875fa7f4d
Date: 2016-11-01 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/1ee875fa7f4d/

Log:	add the renamed cffi build script

diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl_build.py
@@ -0,0 +1,8 @@
+import sys
+from openssl import _cffi_src
+sys.modules['_cffi_src'] = _cffi_src
+#
+from openssl._cffi_src.build_openssl import ffi
+
+if __name__ == '__main__':
+    ffi.compile()

From pypy.commits at gmail.com  Wed Nov  2 06:33:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 03:33:56 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: add failing test,
 clear forwarded for the align_operations list
Message-ID: <5819c114.85951c0a.b163e.0ce1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88060:53810e4e8c5c
Date: 2016-11-02 08:58 +0100
http://bitbucket.org/pypy/pypy/changeset/53810e4e8c5c/

Log:	add failing test, clear forwarded for the align_operations list

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -15,6 +15,7 @@
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 FLAG_ARRAY = 0
 FLAG_STR = 1
@@ -302,7 +303,11 @@
         self._changed_op = None
         for i in range(len(operations)):
             op = operations[i]
-            assert op.get_forwarded() is None
+            if op.get_forwarded():
+                msg = '[rewrite] operations at %d has forwarded info %s\n' % (i, op.repr({}))
+                if we_are_translated():
+                    llop.debug_print(lltype.Void, msg)
+                raise NotImplementedError(msg)
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             if op is self._changed_op:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -61,7 +61,7 @@
                 return False
         return True
 
-    def ensure_operations(self, opstrlist, trace, inthatorder=True):
+    def ensure_operations(self, opstrlist, trace):
         oparse = OpParser('', self.cpu, self.namespace, None,
                           None, True, None)
         oplist = []
@@ -75,7 +75,6 @@
                 oparse._cache['lltype', elem] = op
             oplist.append(op)
         oplist_i = 0
-        match = False
         remap = {}
         last_match = 0
         for i, op in enumerate(trace.operations):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1373,5 +1373,25 @@
             'i2 = int_add(i0, 16)',
         ], trace)
 
+    def test_schedule_signext_twice(self):
+        trace = self.parse_loop("""
+        [p0, i1, p2, i3, p4, p5, i6, i7]
+        i8 = raw_load_i(i6, i3, descr=chararraydescr)
+        i10 = int_signext(i8, 1)
+        guard_not_invalidated() [p2, i10, i3, i1, p0]
+        i11 = int_is_true(i10)
+        guard_false(i11) [p2, i10, i3, i1, p0]
+        i13 = int_add(i1, 1)
+        i15 = int_add(i3, 1)
+        i16 = int_ge(i13, i7)
+        guard_false(i16) [p2, i10, i3, i1, p0]
+        jump(p0, i13, p2, i15, p4, p5, i6, i7)
+        """)
+        self.schedule(trace, unroll_factor=15)
+        dups = set()
+        for op in trace.operations:
+            assert op not in dups
+            dups.add(op)
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vector.py b/rpython/jit/metainterp/optimizeopt/vector.py
--- a/rpython/jit/metainterp/optimizeopt/vector.py
+++ b/rpython/jit/metainterp/optimizeopt/vector.py
@@ -83,7 +83,7 @@
             oplist = self.prefix
         if label:
             oplist = [self.label] + oplist
-        if label != True:
+        if not label:
             for op in oplist:
                 op.set_forwarded(None)
             self.jump.set_forwarded(None)
@@ -264,6 +264,8 @@
         state.schedule()
 
         info.extra_before_label = loop.align_operations
+        for op in loop.align_operations:
+            op.set_forwarded(None)
 
         return loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False)
 

From pypy.commits at gmail.com  Wed Nov  2 06:33:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 03:33:58 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: do not allow duplicates in
 delayed list
Message-ID: <5819c116.0777c20a.532be.9366@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88061:b8e14645ed99
Date: 2016-11-02 09:28 +0100
http://bitbucket.org/pypy/pypy/changeset/b8e14645ed99/

Log:	do not allow duplicates in delayed list

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -138,10 +138,10 @@
     def delay_emit(self, node):
         """ it has been decided that the operation might be scheduled later """
         delayed = node.delayed or []
-        delayed.append(node)
+        if node not in delayed:
+            delayed.append(node)
         node.delayed = None
         provides = node.provides()
-        op = node.getoperation()
         if len(provides) == 0:
             for n in delayed:
                 self.delayed.append(n)
@@ -158,7 +158,8 @@
         else:
             delayedlist = node.delayed
             for d in delayed:
-                delayedlist.append(d)
+                if d not in delayedlist:
+                    delayedlist.append(d)
 
 
     def mark_emitted(state, node, unpack=True):
@@ -347,7 +348,7 @@
             state.renamer.start_renaming(op, vecop)
     if left.is_guard():
         prepare_fail_arguments(state, pack, left, vecop)
-    state.oplist.append(vecop)
+    state.append_to_oplist(vecop)
     assert vecop.count >= 1
 
 def prepare_arguments(state, oprestrict, pack, args):
@@ -414,7 +415,7 @@
         count = vecinfo.count
         vecop = VecOperationNew(rop.VEC_INT_SIGNEXT, [arg, ConstInt(newsize)],
                                 'i', newsize, vecinfo.signed, count)
-        state.oplist.append(vecop)
+        state.append_to_oplist(vecop)
         state.costmodel.record_cast_int(size, newsize, count)
         args[i] = vecop
 
@@ -483,7 +484,7 @@
     vecop = OpHelpers.create_vec_unpack(arg.type, args, vecinfo.bytesize,
                                         vecinfo.signed, count)
     state.costmodel.record_vector_unpack(arg, index, count)
-    state.oplist.append(vecop)
+    state.append_to_oplist(vecop)
     return vecop
 
 def pack_into_vector(state, tgt, tidx, src, sidx, scount):
@@ -496,7 +497,7 @@
     newcount = vecinfo.count + scount
     args = [tgt, src, ConstInt(tidx), ConstInt(scount)]
     vecop = OpHelpers.create_vec_pack(tgt.type, args, vecinfo.bytesize, vecinfo.signed, newcount)
-    state.oplist.append(vecop)
+    state.append_to_oplist(vecop)
     state.costmodel.record_vector_pack(src, sidx, scount)
     if not we_are_translated():
         _check_vec_pack(vecop)
@@ -678,7 +679,6 @@
                 self.pre_emit(node, i==0)
                 self.mark_emitted(node, unpack=False)
             turn_into_vector(self, node.pack)
-            return
         elif not node.emitted:
             SchedulerState.try_emit_or_delay(self, node)
 

From pypy.commits at gmail.com  Wed Nov  2 06:34:17 2016
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 02 Nov 2016 03:34:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: document this branch
Message-ID: <5819c129.0d8d1c0a.fca98.ab9c@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r88062:c8fcf9b35525
Date: 2016-11-02 11:33 +0100
http://bitbucket.org/pypy/pypy/changeset/c8fcf9b35525/

Log:	document this branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -80,3 +80,10 @@
 
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)

From pypy.commits at gmail.com  Wed Nov  2 06:34:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 03:34:51 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <5819c14b.c1d81c0a.f3776.ada2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88063:a51b31b31124
Date: 2016-11-02 11:34 +0100
http://bitbucket.org/pypy/pypy/changeset/a51b31b31124/

Log:	merge default

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -16,6 +16,9 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
+We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+well as maintaining support for previous versions.
+
 XXX
 
 As always, this release fixed many issues and bugs raised by the
@@ -72,7 +75,7 @@
 * New features
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
-  * Backport fixes to rposix on windows from py2.5
+  * Backport fixes to rposix on windows from py3.5
   * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
   * Add ``inode`` to ``scandir()`` on posix systems
   * Support more attributes on ``super``
@@ -98,6 +101,8 @@
   * Support translation on FreeBSD running on PowerPC
   * Implement ``__rmod__`` on ``str`` and ``unicode`` types
   * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.4"
+VERSION = "1.9.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names
+from pypy.module._hashlib.interp_hashlib import (
+    algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC)
 
 
 class Module(MixedModule):
@@ -13,6 +14,9 @@
     for name in algorithms:
         interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
 
+    if HAS_FAST_PKCS5_PBKDF2_HMAC:
+        interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac'
+
     def startup(self, space):
         w_meth_names = fetch_names(space)
         space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -8,7 +8,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.thread.os_lock import Lock
 
@@ -58,23 +58,26 @@
     def __init__(self, space, name, copy_from=NULL_CTX):
         self.name = name
         digest_type = self.digest_type_by_name(space)
-        self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
+        self.digest_size = ropenssl.EVP_MD_size(digest_type)
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
-        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
         rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
         try:
             if copy_from:
-                ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
+                    raise ValueError
             else:
                 ropenssl.EVP_DigestInit(ctx, digest_type)
             self.ctx = ctx
         except:
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
             raise
         self.register_finalizer(space)
 
@@ -82,8 +85,7 @@
         ctx = self.ctx
         if ctx:
             self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-            ropenssl.EVP_MD_CTX_cleanup(ctx)
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
 
     def digest_type_by_name(self, space):
         digest_type = ropenssl.EVP_get_digestbyname(self.name)
@@ -128,21 +130,26 @@
 
     def get_block_size(self, space):
         digest_type = self.digest_type_by_name(space)
-        block_size = rffi.getintfield(digest_type, 'c_block_size')
+        block_size = ropenssl.EVP_MD_block_size(digest_type)
         return space.wrap(block_size)
 
     def get_name(self, space):
         return space.wrap(self.name)
 
     def _digest(self, space):
-        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
+        try:
             with self.lock:
-                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx):
+                    raise ValueError
             digest_size = self.digest_size
             with rffi.scoped_alloc_buffer(digest_size) as buf:
                 ropenssl.EVP_DigestFinal(ctx, buf.raw, None)
-                ropenssl.EVP_MD_CTX_cleanup(ctx)
                 return buf.str(digest_size)
+        finally:
+            ropenssl.EVP_MD_CTX_free(ctx)
 
 
 W_Hash.typedef = TypeDef(
@@ -177,3 +184,27 @@
 for _name in algorithms:
     _newname = 'new_%s' % (_name,)
     globals()[_newname] = make_new_hash(_name, _newname)
+
+
+HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
+if HAS_FAST_PKCS5_PBKDF2_HMAC:
+    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+                 w_dklen=WrappedDefault(None))
+    def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
+        digest = ropenssl.EVP_get_digestbyname(name)
+        if not digest:
+            raise oefmt(space.w_ValueError, "unknown hash function")
+        if space.is_w(w_dklen, space.w_None):
+            dklen = ropenssl.EVP_MD_size(digest)
+        else:
+            dklen = space.int_w(w_dklen)
+        if dklen < 1:
+            raise oefmt(space.w_ValueError,
+                        "key length must be greater than 0.")
+        with rffi.scoped_alloc_buffer(dklen) as buf:
+            r = ropenssl.PKCS5_PBKDF2_HMAC(
+                password, len(password), salt, len(salt), rounds, digest,
+                dklen, buf.raw)
+            if not r:
+                raise ValueError
+            return space.wrap(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -15,17 +15,19 @@
 
     def test_attributes(self):
         import hashlib
-        for name, expected_size in {'md5': 16,
-                                    'sha1': 20,
-                                    'sha224': 28,
-                                    'sha256': 32,
-                                    'sha384': 48,
-                                    'sha512': 64,
-                                    }.items():
+        for name, (expected_size, expected_block_size) in {
+                'md5': (16, 64),
+                'sha1': (20, 64),
+                'sha224': (28, 64),
+                'sha256': (32, 64),
+                'sha384': (48, 128),
+                'sha512': (64, 128),
+                }.items():
             h = hashlib.new(name)
             assert h.name == name
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -46,6 +48,7 @@
             h = py_new(name)('')
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -108,3 +111,13 @@
             got.decode('hex')
             if expected is not None:
                 assert got == expected
+
+    def test_pbkdf2(self):
+        try:
+            from _hashlib import pbkdf2_hmac
+        except ImportError:
+            skip("Requires OpenSSL >= 1.1")
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -33,9 +33,10 @@
 PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
 
 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
+ PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
  PY_SSL_VERSION_TLS1_2) = range(6)
 
+
 SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
 SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
@@ -68,11 +69,12 @@
 constants["HAS_NPN"] = HAS_NPN
 constants["HAS_ALPN"] = HAS_ALPN
 
+constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
+constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 if not OPENSSL_NO_SSL3:
     constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 if HAVE_TLSv1_2:
     constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
@@ -111,7 +113,10 @@
         err_reason = libssl_ERR_GET_REASON(errcode)
         reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
         lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        msg = rffi.charp2str(libssl_ERR_reason_error_string(errcode))
+        raw_msg = libssl_ERR_reason_error_string(errcode)
+        msg = None
+        if raw_msg:
+            msg = rffi.charp2str(raw_msg)
     if not msg:
         msg = "unknown error"
     if reason_str and lib_str:
@@ -637,9 +642,12 @@
         if not self.ssl:
             return space.w_None
         comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method or intmask(comp_method[0].c_type) == NID_undef:
+        if not comp_method:
             return space.w_None
-        short_name = libssl_OBJ_nid2sn(comp_method[0].c_type)
+        method_type = intmask(libssl_COMP_get_type(comp_method))
+        if method_type == NID_undef:
+            return space.w_None
+        short_name = libssl_COMP_get_name(comp_method)
         if not short_name:
             return space.w_None
         return space.wrap(rffi.charp2str(short_name))
@@ -795,7 +803,7 @@
     for index in range(entry_count):
         entry = libssl_X509_NAME_get_entry(xname, index)
         # check to see if we've gotten to a new RDN
-        entry_level = intmask(entry[0].c_set)
+        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
         if rdn_level >= 0:
             if rdn_level != entry_level:
                 # yes, new RDN
@@ -846,8 +854,9 @@
                                 "No method for internalizing subjectAltName!'")
 
             with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                p_ptr[0] = ext[0].c_value.c_data
-                length = intmask(ext[0].c_value.c_length)
+                ext_value = libssl_X509_EXTENSION_get_data(ext)
+                p_ptr[0] = ext_value.c_data
+                length = intmask(ext_value.c_length)
                 null = lltype.nullptr(rffi.VOIDP.TO)
                 if method[0].c_it:
                     names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
@@ -967,10 +976,8 @@
     if OPENSSL_VERSION_NUMBER >= 0x10001000:
         # Calls x509v3_cache_extensions and sets up crldp
         libssl_X509_check_ca(certificate)
-        dps = certificate[0].c_crldp
-    else:
-        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-            certificate, NID_crl_distribution_points, None, None))
+    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+        certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
@@ -1268,14 +1275,14 @@
     @staticmethod
     @unwrap_spec(protocol=int)
     def descr_new(space, w_subtype, protocol):
-        if protocol == PY_SSL_VERSION_TLS1:
+        if protocol == PY_SSL_VERSION_TLS:
+            method = libssl_TLS_method()
+        elif protocol == PY_SSL_VERSION_TLS1:
             method = libssl_TLSv1_method()
         elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
             method = libssl_SSLv3_method()
         elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
             method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_SSL23:
-            method = libssl_SSLv23_method()
         elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
             method = libssl_TLSv1_1_method()
         elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
@@ -1303,6 +1310,7 @@
             # OpenSSL 1.0.2+), or use prime256v1 by default.
             # This is Apache mod_ssl's initialization
             # policy, so we should be safe.
+            # OpenSSL 1.1 has it enabled by default.
             if libssl_SSL_CTX_set_ecdh_auto:
                 libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
@@ -1390,20 +1398,22 @@
 
     def descr_get_verify_flags(self, space):
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         return space.wrap(flags)
 
     def descr_set_verify_flags(self, space, w_obj):
         new_flags = space.int_w(w_obj)
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         flags_clear = flags & ~new_flags
         flags_set = ~flags & new_flags
         if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                store[0].c_param, flags_clear):
+                param, flags_clear):
             raise _ssl_seterror(space, None, 0)
         if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                store[0].c_param, flags_set):
+                param, flags_set):
             raise _ssl_seterror(space, None, 0)
 
     def descr_get_check_hostname(self, space):
@@ -1614,14 +1624,16 @@
         x509 = 0
         x509_ca = 0
         crl = 0
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) == X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
+            if obj_type == X509_LU_X509:
                 x509 += 1
                 if libssl_X509_check_ca(
-                        libssl_pypy_X509_OBJECT_data_x509(obj)):
+                        libssl_X509_OBJECT_get0_X509(obj)):
                     x509_ca += 1
-            elif intmask(obj.c_type) == X509_LU_CRL:
+            elif obj_type == X509_LU_CRL:
                 crl += 1
             else:
                 # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
@@ -1660,13 +1672,14 @@
             binary_mode = False
         rlist = []
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) != X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
                 # not a x509 cert
                 continue
             # CA for any purpose
-            cert = libssl_pypy_X509_OBJECT_data_x509(obj)
+            cert = libssl_X509_OBJECT_get0_X509(obj)
             if not libssl_X509_check_ca(cert):
                 continue
             if binary_mode:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -77,7 +77,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
             ss = ctx._wrap_socket(s, 0)
             assert ss.context is ctx
         exc = raises(_socket.error, ss.do_handshake)
@@ -95,7 +95,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0)
+            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLS)._wrap_socket(s, 0)
         s.close()
         exc = raises(_ssl.SSLError, ss.write, "data")
         assert exc.value.message == 'Underlying socket has been closed.'
@@ -123,13 +123,13 @@
 
     def test_context(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         raises(ValueError, _ssl._SSLContext, -1)
 
         assert type(s.options) is long
-        assert s.options & _ssl.OP_NO_SSLv2
-        s.options &= ~_ssl.OP_NO_SSLv2
-        assert not s.options & _ssl.OP_NO_SSLv2
+        assert s.options & _ssl.OP_NO_SSLv3
+        s.options &= ~_ssl.OP_NO_SSLv3
+        assert not s.options & _ssl.OP_NO_SSLv3
         raises(TypeError, "s.options = 2.5")
 
         assert not s.check_hostname
@@ -159,7 +159,7 @@
 
     def test_set_default_verify_paths(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s.set_default_verify_paths()
 
 
@@ -253,13 +253,44 @@
         if not _ssl.HAS_NPN:
             skip("NPN requires OpenSSL 1.0.1 or greater")
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s._sock, True,
                               server_hostname="svn.python.org")
         self.s.close()
         del ss; gc.collect()
 
+    def test_peer_certificate(self):
+        import _ssl, gc
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ss = ctx._wrap_socket(self.s._sock, False)
+        ss.do_handshake()
+        assert isinstance(ss.peer_certificate(der=True), bytes)
+        assert isinstance(ss.peer_certificate(), dict)
+        self.s.close()
+        del ss; gc.collect()
+
+    def test_peer_certificate_verify(self):
+        import _ssl, ssl, gc
+        paths = ssl.get_default_verify_paths()
+        if not paths.capath and not paths.cafile:
+            skip("ssl.get_default_verify_paths() failed to return any path")
+
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.verify_mode = _ssl.CERT_REQUIRED
+        ctx.load_verify_locations(capath=paths.capath, cafile=paths.cafile)
+
+        ss = ctx._wrap_socket(self.s._sock, False)
+        try:
+            ss.do_handshake()
+        except _ssl.SSLError as e:
+            if e.reason == 'CERTIFICATE_VERIFY_FAILED':
+                skip("Certificate verification failed. "
+                     "Most likely we just don't have any CA certificates.")
+        assert ss.peer_certificate()
+        self.s.close()
+        del ss; gc.collect()
+
     def test_tls_unique_cb(self):
         import ssl, sys, gc
         ss = ssl.wrap_socket(self.s)
@@ -325,7 +356,7 @@
 
     def test_load_cert_chain(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
         exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
@@ -344,11 +375,11 @@
 
     def test_load_verify_locations(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         ctx.load_verify_locations(cafile=self.keycert, capath=None)
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         with open(self.keycert) as f:
             cacert_pem = f.read().decode('ascii')
         ctx.load_verify_locations(cadata=cacert_pem)
@@ -356,7 +387,7 @@
 
     def test_get_ca_certs(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         assert ctx.get_ca_certs() == []
         ctx.load_verify_locations(self.python_org_cert)
@@ -370,7 +401,7 @@
 
     def test_cert_store_stats(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
         ctx.load_cert_chain(self.keycert)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
@@ -379,7 +410,7 @@
 
     def test_load_dh_params(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
@@ -389,7 +420,7 @@
 
     def test_set_ecdh_curve(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.set_ecdh_curve("prime256v1")
         raises(ValueError, ctx.set_ecdh_curve, "foo")
 
@@ -434,7 +465,7 @@
     def test_lib_reason(self):
         # Test the library and reason attributes
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         exc = raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
         assert exc.value.library == 'PEM'
         assert exc.value.reason == 'NO_START_LINE'
@@ -445,7 +476,7 @@
         # Check that the appropriate SSLError subclass is raised
         # (this only tests one of them)
         import _ssl, _socket
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s = _socket.socket()
         try:
             s.bind(("127.0.0.1", 0))
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -524,8 +524,8 @@
             maxindex = XML_GetSpecifiedAttributeCount(self.itself)
         else:
             maxindex = 0
-        while attrs[maxindex]:
-            maxindex += 2 # copied
+            while attrs[maxindex]:
+                maxindex += 2 # copied
 
         if self.ordered_attributes:
             w_attrs = space.newlist([
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -57,8 +57,19 @@
         '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)',
         '#define pypy_GENERAL_NAME_uri(name) (name->d.uniformResourceIdentifier)',
         '#define pypy_GENERAL_NAME_pop_free(names) (sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free))',
-        '#define pypy_X509_OBJECT_data_x509(obj) (obj->data.x509)',
         '#define pypy_DIST_POINT_fullname(obj) (obj->distpoint->name.fullname)',
+        # Backwards compatibility for functions introduced in 1.1
+        '#if (OPENSSL_VERSION_NUMBER < 0x10100000) || defined(LIBRESSL_VERSION_NUMBER)\n'
+        '#  define COMP_get_name(meth) (meth->name)\n'
+        '#  define COMP_get_type(meth) (meth->type)\n'
+        '#  define X509_NAME_ENTRY_set(ne) (ne->set)\n'
+        '#  define X509_OBJECT_get0_X509(obj) (obj->data.x509)\n'
+        '#  define X509_OBJECT_get_type(obj) (obj->type)\n'
+        '#  define X509_STORE_get0_objects(store) (store->objs)\n'
+        '#  define X509_STORE_get0_param(store) (store->param)\n'
+        '#else\n'
+        '#  define OPENSSL_NO_SSL2\n'
+        '#endif',
     ],
 )
 
@@ -88,6 +99,7 @@
             "OPENSSL_EXPORT_VAR_AS_FUNCTION")
     OPENSSL_VERSION_NUMBER = rffi_platform.ConstantInteger(
         "OPENSSL_VERSION_NUMBER")
+    LIBRESSL = rffi_platform.Defined("LIBRESSL_VERSION_NUMBER")
 
 cconfig = rffi_platform.configure(CConfigBootstrap)
 if cconfig["OPENSSL_EXPORT_VAR_AS_FUNCTION"]:
@@ -95,23 +107,10 @@
 else:
     ASN1_ITEM_EXP = ASN1_ITEM
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
+LIBRESSL = cconfig["LIBRESSL"]
+OPENSSL_1_1 = OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
-     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
-    eci.pre_include_bits = ()
-    eci.post_include_bits = ()
-    raise Exception("""OpenSSL version >= 1.1 not supported yet.
-
-    This program requires OpenSSL version 1.0.x, and may also
-    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
-    some work to update to; contributions are welcome.  Sorry,
-    you need to install an older version of OpenSSL for now.
-    Make sure this older version is the one picked up by this
-    program when it runs the compiler.
-    
-    This is the configuration used: %r""" % (eci,))
-
 
 class CConfig:
     _compilation_info_ = eci
@@ -210,28 +209,10 @@
     OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger(
         "OBJ_NAME_TYPE_MD_METH")
 
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        X509_st = rffi_platform.Struct(
-            'struct x509_st',
-            [('crldp', stack_st_DIST_POINT)])
-
     # Some structures, with only the fields used in the _ssl module
-    X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st',
-                                              [('set', rffi.INT)])
     asn1_string_st = rffi_platform.Struct('struct asn1_string_st',
                                           [('length', rffi.INT),
                                            ('data', rffi.CCHARP)])
-    X509_extension_st = rffi_platform.Struct(
-        'struct X509_extension_st',
-        [('value', ASN1_STRING)])
-    x509_store_st = rffi_platform.Struct(
-        'struct x509_store_st',
-        [('objs', stack_st_X509_OBJECT),
-         ('param', X509_VERIFY_PARAM)])
-    x509_object_st = rffi_platform.Struct(
-        'struct x509_object_st',
-        [('type', rffi.INT)])
-
     X509_LU_X509 = rffi_platform.ConstantInteger("X509_LU_X509")
     X509_LU_CRL = rffi_platform.ConstantInteger("X509_LU_CRL")
 
@@ -244,12 +225,6 @@
     GENERAL_NAME_st = rffi_platform.Struct(
         'struct GENERAL_NAME_st',
         [('type', rffi.INT)])
-    EVP_MD_st = rffi_platform.Struct(
-        'EVP_MD',
-        [('md_size', rffi.INT),
-         ('block_size', rffi.INT)])
-    EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
-    EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
     OBJ_NAME_st = rffi_platform.Struct(
         'OBJ_NAME',
@@ -257,10 +232,6 @@
          ('name', rffi.CCHARP),
          ])
 
-    COMP_METHOD_st = rffi_platform.Struct(
-        'struct comp_method_st',
-        [('type', rffi.INT),])
-
     ACCESS_DESCRIPTION_st = rffi_platform.Struct(
         'struct ACCESS_DESCRIPTION_st',
         [('method', ASN1_OBJECT),
@@ -275,14 +246,12 @@
 SSL_CIPHER = rffi.COpaquePtr('SSL_CIPHER')
 SSL = rffi.COpaquePtr('SSL')
 BIO = rffi.COpaquePtr('BIO')
-if OPENSSL_VERSION_NUMBER >= 0x10001000:
-    X509 = rffi.CArrayPtr(X509_st)
-else:
-    X509 = rffi.COpaquePtr('X509')
-X509_NAME_ENTRY = rffi.CArrayPtr(X509_name_entry_st)
-X509_EXTENSION = rffi.CArrayPtr(X509_extension_st)
-X509_STORE = rffi.CArrayPtr(x509_store_st)
-X509_OBJECT = lltype.Ptr(x509_object_st)
+X509 = rffi.COpaquePtr('X509')
+X509_OBJECT = rffi.COpaquePtr('X509_OBJECT')
+COMP_METHOD = rffi.COpaquePtr('COMP_METHOD')
+X509_NAME_ENTRY = rffi.COpaquePtr('X509_NAME_ENTRY')
+X509_EXTENSION = rffi.COpaquePtr('X509_EXTENSION')
+X509_STORE = rffi.COpaquePtr('X509_STORE')
 X509V3_EXT_METHOD = rffi.CArrayPtr(v3_ext_method)
 ASN1_STRING.TO.become(asn1_string_st)
 ASN1_TIME = rffi.COpaquePtr('ASN1_TIME')
@@ -290,9 +259,9 @@
 GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES')
 GENERAL_NAME.TO.become(GENERAL_NAME_st)
 OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
-COMP_METHOD = rffi.CArrayPtr(COMP_METHOD_st)
 ACCESS_DESCRIPTION = rffi.CArrayPtr(ACCESS_DESCRIPTION_st)
 
+
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_OPENSSL_FINISHED = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df and \
@@ -314,8 +283,10 @@
     globals()['libssl_' + name] = external(
         name, argtypes, restype, **kw)
 
-ssl_external('SSL_load_error_strings', [], lltype.Void)
-ssl_external('SSL_library_init', [], rffi.INT)
+ssl_external('SSL_load_error_strings', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
+ssl_external('SSL_library_init', [], rffi.INT,
+    macro=OPENSSL_1_1 or None)
 ssl_external('CRYPTO_num_locks', [], rffi.INT)
 ssl_external('CRYPTO_set_locking_callback',
              [lltype.Ptr(lltype.FuncType(
@@ -341,7 +312,11 @@
     ssl_external('TLSv1_2_method', [], SSL_METHOD)
 ssl_external('SSLv2_method', [], SSL_METHOD)
 ssl_external('SSLv3_method', [], SSL_METHOD)
-ssl_external('SSLv23_method', [], SSL_METHOD)
+# Windows note: fails in untranslated tests if the following function is
+# made 'macro=True'.  Not sure I want to dig into the reason for that mess.
+libssl_TLS_method = external(
+    'TLS_method' if OPENSSL_1_1 else 'SSLv23_method',
+    [], SSL_METHOD)
 ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT,
@@ -370,7 +345,7 @@
              lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tmp_ecdh', [SSL_CTX, EC_KEY], lltype.Void, macro=True)
-if OPENSSL_VERSION_NUMBER >= 0x10002000:
+if OPENSSL_VERSION_NUMBER >= 0x10002000 and not OPENSSL_1_1:
     ssl_external('SSL_CTX_set_ecdh_auto', [SSL_CTX, rffi.INT], lltype.Void,
                  macro=True)
 else:
@@ -416,6 +391,8 @@
 ssl_external('X509_NAME_get_entry', [X509_NAME, rffi.INT], X509_NAME_ENTRY)
 ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT)
 ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING)
+ssl_external('X509_NAME_ENTRY_set', [X509_NAME_ENTRY], rffi.INT,
+    macro=(not OPENSSL_1_1) or None)
 ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT, save_err=SAVE_ERR)
 ssl_external('X509_free', [X509], lltype.Void, releasegil=False)
 ssl_external('X509_check_ca', [X509], rffi.INT)
@@ -427,11 +404,16 @@
 ssl_external('X509_get_ext', [X509, rffi.INT], X509_EXTENSION)
 ssl_external('X509_get_ext_d2i', [X509, rffi.INT, rffi.VOIDP, rffi.VOIDP], rffi.VOIDP)
 ssl_external('X509V3_EXT_get', [X509_EXTENSION], X509V3_EXT_METHOD)
+ssl_external('X509_EXTENSION_get_data', [X509_EXTENSION], ASN1_STRING)
 
 ssl_external('X509_VERIFY_PARAM_get_flags', [X509_VERIFY_PARAM], rffi.ULONG)
 ssl_external('X509_VERIFY_PARAM_set_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_VERIFY_PARAM_clear_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_STORE_add_cert', [X509_STORE, X509], rffi.INT)
+ssl_external('X509_STORE_get0_objects', [X509_STORE], stack_st_X509_OBJECT,
+    macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_STORE_get0_param', [X509_STORE], X509_VERIFY_PARAM,
+    macro=bool(not OPENSSL_1_1) or None)
 
 ssl_external('X509_get_default_cert_file_env', [], rffi.CCHARP)
 ssl_external('X509_get_default_cert_file', [], rffi.CCHARP)
@@ -468,8 +450,14 @@
              macro=True)
 ssl_external('sk_X509_OBJECT_value', [stack_st_X509_OBJECT, rffi.INT],
              X509_OBJECT, macro=True)
-ssl_external('pypy_X509_OBJECT_data_x509', [X509_OBJECT], X509,
-             macro=True)
+ssl_external('X509_OBJECT_get0_X509', [X509_OBJECT], X509,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_OBJECT_get_type', [X509_OBJECT], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_name', [COMP_METHOD], rffi.CCHARP,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_type', [COMP_METHOD], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('sk_DIST_POINT_num', [stack_st_DIST_POINT], rffi.INT,
              macro=True)
 ssl_external('sk_DIST_POINT_value', [stack_st_DIST_POINT, rffi.INT], DIST_POINT,
@@ -511,8 +499,12 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
-libssl_OPENSSL_free = libssl_CRYPTO_free
+if OPENSSL_1_1:
+    ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
+else:
+    ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
+    libssl_OPENSSL_free = libssl_CRYPTO_free
+    del libssl_CRYPTO_free
 
 ssl_external('SSL_write', [SSL, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=SAVE_ERR)
@@ -576,10 +568,11 @@
             SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = lltype.Ptr(EVP_MD_st)
+EVP_MD = rffi.COpaquePtr('EVP_MD')
 
 OpenSSL_add_all_digests = external(
-    'OpenSSL_add_all_digests', [], lltype.Void)
+    'OpenSSL_add_all_digests', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
 EVP_get_digestbyname = external(
     'EVP_get_digestbyname',
     [rffi.CCHARP], EVP_MD)
@@ -592,10 +585,25 @@
 EVP_DigestFinal = external(
     'EVP_DigestFinal',
     [EVP_MD_CTX, rffi.CCHARP, rffi.VOIDP], rffi.INT)
+EVP_MD_size = external(
+    'EVP_MD_size', [EVP_MD], rffi.INT)
+EVP_MD_block_size = external(
+    'EVP_MD_block_size', [EVP_MD], rffi.INT)
 EVP_MD_CTX_copy = external(
     'EVP_MD_CTX_copy', [EVP_MD_CTX, EVP_MD_CTX], rffi.INT)
-EVP_MD_CTX_cleanup = external(
-    'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, releasegil=False)
+EVP_MD_CTX_new = external(
+    'EVP_MD_CTX_new' if OPENSSL_1_1 else 'EVP_MD_CTX_create',
+    [], EVP_MD_CTX)
+EVP_MD_CTX_free = external(
+    'EVP_MD_CTX_free' if OPENSSL_1_1 else 'EVP_MD_CTX_destroy',
+    [EVP_MD_CTX], lltype.Void, releasegil=False)
+
+if OPENSSL_1_1:
+    PKCS5_PBKDF2_HMAC = external('PKCS5_PBKDF2_HMAC', [
+        rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT, rffi.INT, EVP_MD,
+        rffi.INT, rffi.CCHARP], rffi.INT)
+else:
+    PKCS5_PBKDF2_HMAC = None
 
 OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType(
         [OBJ_NAME, rffi.VOIDP], lltype.Void))
@@ -606,7 +614,7 @@
 # Used for adding memory pressure. Last number is an (under?)estimate of
 # EVP_PKEY_CTX's size.
 # XXX: Make a better estimate here
-HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \
+HASH_MALLOC_SIZE = 120 + 48 \
         + rffi.sizeof(EVP_MD) * 2 + 208
 
 def init_ssl():
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1575,15 +1575,13 @@
 
         class A:
             def __init__(self):
-                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
-                    flavor='raw')
                 digest = ropenssl.EVP_get_digestbyname('sha1')
+                self.ctx = ropenssl.EVP_MD_CTX_new()
                 ropenssl.EVP_DigestInit(self.ctx, digest)
                 rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
 
             def __del__(self):
-                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
-                lltype.free(self.ctx, flavor='raw')
+                ropenssl.EVP_MD_CTX_free(self.ctx)
         #A() --- can't call it here?? get glibc crashes on tannit64
         def f():
             am1 = am2 = am3 = None

From pypy.commits at gmail.com  Wed Nov  2 06:51:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Nov 2016 03:51:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: document branches
Message-ID: <5819c53e.0370c20a.2d5a9.9a37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88064:cf1fca4da201
Date: 2016-11-02 11:51 +0100
http://bitbucket.org/pypy/pypy/changeset/cf1fca4da201/

Log:	document branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -87,3 +87,13 @@
 ``struct.unpack`` now returns an ``int`` whenever the returned value fits,
 while previously it always returned a ``long`` for certains format codes such
 as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: newinitwarn
+
+Issue warnings for stricter handling of ``__new/init__`` args (that
+become TypeErrors in python 3)
+
+.. branch: openssl-1.1
+
+PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
+installed.  Thanks tumbleweed!

From pypy.commits at gmail.com  Wed Nov  2 08:41:25 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 05:41:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: add two missing docstrings
Message-ID: <5819def5.e39cc20a.e2b77.c77a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88065:35d29fcad35b
Date: 2016-11-02 13:40 +0100
http://bitbucket.org/pypy/pypy/changeset/35d29fcad35b/

Log:	add two missing docstrings

	(they were missing in CPython 2 as well, but CPython 3 added them).

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -479,6 +479,14 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recv_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recv_into(buffer, [nbytes[, flags]]) -> nbytes_read
+
+        A version of recv() that stores its data into a buffer rather than creating
+        a new string.  Receive up to buffersize bytes from the socket.  If buffersize
+        is not specified (or 0), receive up to the size available in the given buffer.
+
+        See recv() for documentation about the flags.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0 or nbytes > lgt:
@@ -490,6 +498,10 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recvfrom_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)
+
+        Like recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0:

From pypy.commits at gmail.com  Wed Nov  2 09:03:33 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:33 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: sys module
Message-ID: <5819e425.88571c0a.fa342.65a4@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88066:9cd4b09a39f8
Date: 2016-11-02 10:38 +0100
http://bitbucket.org/pypy/pypy/changeset/9cd4b09a39f8/

Log:	sys module

diff --git a/pypy/module/parser/__init__.py b/pypy/module/parser/__init__.py
--- a/pypy/module/parser/__init__.py
+++ b/pypy/module/parser/__init__.py
@@ -10,8 +10,8 @@
     }
 
     interpleveldefs = {
-        '__name__'     : '(space.wrap("parser"))',
-        '__doc__'      : '(space.wrap("parser module"))',
+        '__name__'     : '(space.newtext("parser"))',
+        '__doc__'      : '(space.newtext("parser module"))',
         'suite'        : 'pyparser.suite',
         'expr'         : 'pyparser.expr',
         'issuite'      : 'pyparser.issuite',
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -24,21 +24,21 @@
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
-        '__name__'              : '(space.wrap("sys"))',
-        '__doc__'               : '(space.wrap("PyPy sys module"))',
+        '__name__'              : '(space.newtext("sys"))',
+        '__doc__'               : '(space.newtext("PyPy sys module"))',
 
-        'platform'              : 'space.wrap(sys.platform)',
-        'maxint'                : 'space.wrap(sys.maxint)',
-        'maxsize'               : 'space.wrap(sys.maxint)',
-        'byteorder'             : 'space.wrap(sys.byteorder)',
-        'maxunicode'            : 'space.wrap(vm.MAXUNICODE)',
+        'platform'              : 'space.newtext(sys.platform)',
+        'maxint'                : 'space.newint(sys.maxint)',
+        'maxsize'               : 'space.newint(sys.maxint)',
+        'byteorder'             : 'space.newtext(sys.byteorder)',
+        'maxunicode'            : 'space.newint(vm.MAXUNICODE)',
         'stdin'                 : 'state.getio(space).w_stdin',
         '__stdin__'             : 'state.getio(space).w_stdin',
         'stdout'                : 'state.getio(space).w_stdout',
         '__stdout__'            : 'state.getio(space).w_stdout',
         'stderr'                : 'state.getio(space).w_stderr',
         '__stderr__'            : 'state.getio(space).w_stderr',
-        'pypy_objspaceclass'    : 'space.wrap(repr(space))',
+        'pypy_objspaceclass'    : 'space.newtext(repr(space))',
         #'prefix'               : # added by pypy_initial_path() when it
         #'exec_prefix'          : # succeeds, pointing to trunk or /usr
         'path'                  : 'state.get(space).w_path',
@@ -79,10 +79,10 @@
 
         'displayhook'           : 'hook.displayhook',
         '__displayhook__'       : 'hook.__displayhook__',
-        'meta_path'             : 'space.wrap([])',
-        'path_hooks'            : 'space.wrap([])',
-        'path_importer_cache'   : 'space.wrap({})',
-        'dont_write_bytecode'   : 'space.wrap(space.config.translation.sandbox)',
+        'meta_path'             : 'space.newlist([])',
+        'path_hooks'            : 'space.newlist([])',
+        'path_importer_cache'   : 'space.newdict()',
+        'dont_write_bytecode'   : 'space.newbool(space.config.translation.sandbox)',
 
         'getdefaultencoding'    : 'interp_encoding.getdefaultencoding',
         'setdefaultencoding'    : 'interp_encoding.setdefaultencoding',
@@ -116,18 +116,18 @@
 
         else:
             from pypy.module.sys import version
-            space.setitem(self.w_dict, space.wrap("version"),
-                          space.wrap(version.get_version(space)))
+            space.setitem(self.w_dict, space.newtext("version"),
+                          version.get_version(space))
             if _WIN:
                 from pypy.module.sys import vm
                 w_handle = vm.get_dllhandle(space)
-                space.setitem(self.w_dict, space.wrap("dllhandle"), w_handle)
+                space.setitem(self.w_dict, space.newtext("dllhandle"), w_handle)
 
     def getmodule(self, name):
         space = self.space
         w_modules = self.get('modules')
         try:
-            return space.getitem(w_modules, space.wrap(name))
+            return space.getitem(w_modules, space.newtext(name))
         except OperationError as e:
             if not e.match(space, space.w_KeyError):
                 raise
@@ -135,7 +135,7 @@
 
     def setmodule(self, w_module):
         space = self.space
-        w_name = self.space.getattr(w_module, space.wrap('__name__'))
+        w_name = self.space.getattr(w_module, space.newtext('__name__'))
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
@@ -161,7 +161,7 @@
             if operror is None:
                 return space.w_None
             else:
-                return space.wrap(operror.get_traceback())
+                return operror.get_traceback()
         return None
 
     def get_w_default_encoder(self):
@@ -176,7 +176,7 @@
 
     def get_flag(self, name):
         space = self.space
-        return space.int_w(space.getattr(self.get('flags'), space.wrap(name)))
+        return space.int_w(space.getattr(self.get('flags'), space.newtext(name)))
 
     def get_state(self, space):
         from pypy.module.sys import state
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
--- a/pypy/module/sys/currentframes.py
+++ b/pypy/module/sys/currentframes.py
@@ -61,17 +61,17 @@
         else:
             frames.append(None)
 
-        w_topframe = space.wrap(None)
+        w_topframe = space.w_None
         w_prevframe = None
         for f in frames:
             w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
             if w_prevframe is None:
                 w_topframe = w_nextframe
             else:
-                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+                space.setattr(w_prevframe, space.newtext('f_back'), w_nextframe)
             w_prevframe = w_nextframe
 
         space.setitem(w_result,
-                      space.wrap(thread_ident),
+                      space.newint(thread_ident),
                       w_topframe)
     return w_result
diff --git a/pypy/module/sys/hook.py b/pypy/module/sys/hook.py
--- a/pypy/module/sys/hook.py
+++ b/pypy/module/sys/hook.py
@@ -3,7 +3,7 @@
 def displayhook(space, w_obj):
     """Print an object to sys.stdout and also save it in __builtin__._"""
     if not space.is_w(w_obj, space.w_None): 
-        space.setitem(space.builtin.w_dict, space.wrap('_'), w_obj)
+        space.setitem(space.builtin.w_dict, space.newtext('_'), w_obj)
         # NB. this is slightly more complicated in CPython,
         # see e.g. the difference with  >>> print 5,; 8
         print_item_to(space, space.repr(w_obj), sys_stdout(space))
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
--- a/pypy/module/sys/initpath.py
+++ b/pypy/module/sys/initpath.py
@@ -117,7 +117,7 @@
 
     if state is not None:    # 'None' for testing only
         lib_extensions = os.path.join(lib_pypy, '__extensions__')
-        state.w_lib_extensions = state.space.wrap(lib_extensions)
+        state.w_lib_extensions = state.space.newtext(lib_extensions)
         importlist.append(lib_extensions)
 
     importlist.append(lib_pypy)
@@ -149,12 +149,12 @@
 
 @unwrap_spec(executable='str0')
 def pypy_find_executable(space, executable):
-    return space.wrap(find_executable(executable))
+    return space.newtext(find_executable(executable))
 
 
 @unwrap_spec(filename='str0')
 def pypy_resolvedirof(space, filename):
-    return space.wrap(resolvedirof(filename))
+    return space.newtext(resolvedirof(filename))
 
 
 @unwrap_spec(executable='str0')
@@ -171,10 +171,10 @@
                 path, prefix = find_stdlib(get_state(space), dyn_path)
         if path is None:
             return space.w_None
-    w_prefix = space.wrap(prefix)
-    space.setitem(space.sys.w_dict, space.wrap('prefix'), w_prefix)
-    space.setitem(space.sys.w_dict, space.wrap('exec_prefix'), w_prefix)
-    return space.newlist([space.wrap(p) for p in path])
+    w_prefix = space.newtext(prefix)
+    space.setitem(space.sys.w_dict, space.newtext('prefix'), w_prefix)
+    space.setitem(space.sys.w_dict, space.newtext('exec_prefix'), w_prefix)
+    return space.newlist([space.newtext(p) for p in path])
 
 
 # ____________________________________________________________
diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py
--- a/pypy/module/sys/interp_encoding.py
+++ b/pypy/module/sys/interp_encoding.py
@@ -5,14 +5,14 @@
 def getdefaultencoding(space):
     """Return the current default string encoding used by the Unicode
 implementation."""
-    return space.wrap(space.sys.defaultencoding)
+    return space.newtext(space.sys.defaultencoding)
 
 def setdefaultencoding(space, w_encoding):
     """Set the current default string encoding used by the Unicode
 implementation."""
     encoding = space.str_w(w_encoding)
     mod = space.getbuiltinmodule("_codecs")
-    w_lookup = space.getattr(mod, space.wrap("lookup"))
+    w_lookup = space.getattr(mod, space.newtext("lookup"))
     # check whether the encoding is there
     space.call_function(w_lookup, w_encoding)
     space.sys.w_default_encoder = None
@@ -21,11 +21,11 @@
 def get_w_default_encoder(space):
     assert not (space.config.translating and not we_are_translated()), \
         "get_w_default_encoder() should not be called during translation"
-    w_encoding = space.wrap(space.sys.defaultencoding)
+    w_encoding = space.newtext(space.sys.defaultencoding)
     mod = space.getbuiltinmodule("_codecs")
-    w_lookup = space.getattr(mod, space.wrap("lookup"))
+    w_lookup = space.getattr(mod, space.newtext("lookup"))
     w_functuple = space.call_function(w_lookup, w_encoding)
-    w_encoder = space.getitem(w_functuple, space.wrap(0))
+    w_encoder = space.getitem(w_functuple, space.newint(0))
     space.sys.w_default_encoder = w_encoder    # cache it
     return w_encoder
 
@@ -51,7 +51,7 @@
                 if loc_codeset:
                     codecmod = space.getbuiltinmodule('_codecs')
                     w_res = space.call_method(codecmod, 'lookup',
-                                              space.wrap(loc_codeset))
+                                              space.newtext(loc_codeset))
                     if space.is_true(w_res):
                         encoding = loc_codeset
             finally:
@@ -66,4 +66,4 @@
     """
     if space.sys.filesystemencoding is None:
         space.sys.filesystemencoding = _getfilesystemencoding(space)
-    return space.wrap(space.sys.filesystemencoding)
+    return space.newtext(space.sys.filesystemencoding)
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -22,7 +22,7 @@
         # Initialize the default path
         srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
-        self.w_path = space.newlist([space.wrap(p) for p in path])
+        self.w_path = space.newlist([space.newtext(p) for p in path])
 
 def get(space):
     return space.fromcache(State)
@@ -33,22 +33,22 @@
         from pypy.module._file.interp_file import W_File
         self.space = space
 
-        stdin = W_File(space)
-        stdin.file_fdopen(0, "r", 1)
-        stdin.w_name = space.wrap('<stdin>')
-        self.w_stdin = space.wrap(stdin)
+        w_stdin = W_File(space)
+        w_stdin.file_fdopen(0, "r", 1)
+        w_stdin.w_name = space.newtext('<stdin>')
+        self.w_stdin = w_stdin
 
-        stdout = W_File(space)
-        stdout.file_fdopen(1, "w", 1)
-        stdout.w_name = space.wrap('<stdout>')
-        self.w_stdout = space.wrap(stdout)
+        w_stdout = W_File(space)
+        w_stdout.file_fdopen(1, "w", 1)
+        w_stdout.w_name = space.newtext('<stdout>')
+        self.w_stdout = w_stdout
 
-        stderr = W_File(space)
-        stderr.file_fdopen(2, "w", 0)
-        stderr.w_name = space.wrap('<stderr>')
-        self.w_stderr = space.wrap(stderr)
+        w_stderr = W_File(space)
+        w_stderr.file_fdopen(2, "w", 0)
+        w_stderr.w_name = space.newtext('<stderr>')
+        self.w_stderr = w_stderr
 
-        stdin._when_reading_first_flush(stdout)
+        w_stdin._when_reading_first_flush(w_stdout)
 
 def getio(space):
     return space.fromcache(IOState)
@@ -58,4 +58,4 @@
     """NOT_RPYTHON
     (should be removed from interpleveldefs before translation)"""
     from rpython.tool.udir import udir
-    return space.wrap(str(udir))
+    return space.newtext(str(udir))
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -31,17 +31,17 @@
 
 def get_float_info(space):
     info_w = [
-        space.wrap(rfloat.DBL_MAX),
-        space.wrap(rfloat.DBL_MAX_EXP),
-        space.wrap(rfloat.DBL_MAX_10_EXP),
-        space.wrap(rfloat.DBL_MIN),
-        space.wrap(rfloat.DBL_MIN_EXP),
-        space.wrap(rfloat.DBL_MIN_10_EXP),
-        space.wrap(rfloat.DBL_DIG),
-        space.wrap(rfloat.DBL_MANT_DIG),
-        space.wrap(rfloat.DBL_EPSILON),
-        space.wrap(rfloat.FLT_RADIX),
-        space.wrap(rfloat.FLT_ROUNDS),
+        space.newfloat(rfloat.DBL_MAX),
+        space.newint(rfloat.DBL_MAX_EXP),
+        space.newint(rfloat.DBL_MAX_10_EXP),
+        space.newfloat(rfloat.DBL_MIN),
+        space.newint(rfloat.DBL_MIN_EXP),
+        space.newint(rfloat.DBL_MIN_10_EXP),
+        space.newint(rfloat.DBL_DIG),
+        space.newint(rfloat.DBL_MANT_DIG),
+        space.newfloat(rfloat.DBL_EPSILON),
+        space.newint(rfloat.FLT_RADIX),
+        space.newint(rfloat.FLT_ROUNDS),
     ]
     w_float_info = app.wget(space, "float_info")
     return space.call_function(w_float_info, space.newtuple(info_w))
@@ -50,17 +50,17 @@
     bits_per_digit = rbigint.SHIFT
     sizeof_digit = rffi.sizeof(rbigint.STORE_TYPE)
     info_w = [
-        space.wrap(bits_per_digit),
-        space.wrap(sizeof_digit),
+        space.newint(bits_per_digit),
+        space.newint(sizeof_digit),
     ]
     w_long_info = app.wget(space, "long_info")
     return space.call_function(w_long_info, space.newtuple(info_w))
 
 def get_float_repr_style(space):
-    return space.wrap("short")
+    return space.newtext("short")
 
 def getdlopenflags(space):
-    return space.wrap(space.sys.dlopenflags)
+    return space.newint(space.sys.dlopenflags)
 
 def setdlopenflags(space, w_flags):
     space.sys.dlopenflags = space.int_w(w_flags)
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -42,10 +42,11 @@
 ''')
 
 def get_api_version(space):
-    return space.wrap(CPYTHON_API_VERSION)
+    return space.newint(CPYTHON_API_VERSION)
 
 def get_version_info(space):
     w_version_info = app.wget(space, "version_info")
+    # run at translation time
     return space.call_function(w_version_info, space.wrap(CPYTHON_VERSION))
 
 def _make_version_template(PYPY_VERSION=PYPY_VERSION):
@@ -65,31 +66,33 @@
 _VERSION_TEMPLATE = _make_version_template()
 
 def get_version(space):
-    return space.wrap(_VERSION_TEMPLATE % compilerinfo.get_compiler_info())
+    return space.newtext(_VERSION_TEMPLATE % compilerinfo.get_compiler_info())
 
 def get_winver(space):
-    return space.wrap("%d.%d" % (
+    return space.newtext("%d.%d" % (
         CPYTHON_VERSION[0],
         CPYTHON_VERSION[1]))
 
 def get_hexversion(space):
-    return space.wrap(tuple2hex(CPYTHON_VERSION))
+    return space.newint(tuple2hex(CPYTHON_VERSION))
 
 def get_pypy_version_info(space):
     ver = PYPY_VERSION
     w_version_info = app.wget(space, "version_info")
+    # run at translation time
     return space.call_function(w_version_info, space.wrap(ver))
 
 def get_subversion_info(space):
+    # run at translation time
     return space.wrap(('PyPy', '', ''))
 
 def get_repo_info(space):
     info = get_repo_version_info(root=pypyroot)
     if info:
         repo_tag, repo_version = info
-        return space.newtuple([space.wrap('PyPy'),
-                               space.wrap(repo_tag),
-                               space.wrap(repo_version)])
+        return space.newtuple([space.newtext('PyPy'),
+                               space.newtext(repo_tag),
+                               space.newtext(repo_version)])
     else:
         return space.w_None
 
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -36,7 +36,7 @@
             raise oefmt(space.w_ValueError, "call stack is not deep enough")
         if depth == 0:
             f.mark_as_escaped()
-            return space.wrap(f)
+            return f
         depth -= 1
         f = ec.getnextframe_nohidden(f)
 
@@ -59,14 +59,14 @@
 def getrecursionlimit(space):
     """Return the last value set by setrecursionlimit().
     """
-    return space.wrap(space.sys.recursionlimit)
+    return space.newint(space.sys.recursionlimit)
 
 @unwrap_spec(flag=bool)
 def set_track_resources(space, flag):
     space.sys.track_resources = flag
 
 def get_track_resources(space):
-    return space.wrap(space.sys.track_resources)
+    return space.newbool(space.sys.track_resources)
 
 @unwrap_spec(interval=int)
 def setcheckinterval(space, interval):
@@ -84,7 +84,7 @@
     result = space.actionflag.getcheckinterval()
     if result <= 1:
         result = 0
-    return space.wrap(result)
+    return space.newint(result)
 
 def exc_info(space):
     """Return the (type, value, traceback) of the most recent exception
@@ -98,7 +98,7 @@
         return space.newtuple([space.w_None, space.w_None, space.w_None])
     else:
         return space.newtuple([operror.w_type, operror.get_w_value(space),
-                               space.wrap(operror.get_traceback())])
+                               operror.get_traceback()])
 
 def exc_info_without_tb(space, frame):
     operror = frame.last_exception
@@ -235,7 +235,7 @@
 @jit.dont_look_inside
 def get_dllhandle(space):
     if not space.config.objspace.usemodules.cpyext:
-        return space.wrap(0)
+        return space.newint(0)
 
     return _get_dllhandle(space)
 
@@ -248,10 +248,10 @@
     # from pypy.module._rawffi.interp_rawffi import W_CDLL
     # from rpython.rlib.clibffi import RawCDLL
     # cdll = RawCDLL(handle)
-    # return space.wrap(W_CDLL(space, "python api", cdll))
+    # return W_CDLL(space, "python api", cdll)
     # Provide a cpython-compatible int
     from rpython.rtyper.lltypesystem import lltype, rffi
-    return space.wrap(rffi.cast(lltype.Signed, handle))
+    return space.newint(rffi.cast(lltype.Signed, handle))
 
 getsizeof_missing = """sys.getsizeof() is not implemented on PyPy.
 

From pypy.commits at gmail.com  Wed Nov  2 09:03:35 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:35 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _collections, _file, _sha, bz2
Message-ID: <5819e427.cf9a1c0a.cc318.62f5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88067:7ece5e8fb99c
Date: 2016-11-02 12:04 +0100
http://bitbucket.org/pypy/pypy/changeset/7ece5e8fb99c/

Log:	_collections, _file, _sha, bz2

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -20,7 +20,7 @@
         char = __builtin__.chr(space.int_w(w_ascii))
     except ValueError:  # chr(out-of-range)
         raise oefmt(space.w_ValueError, "character code not in range(256)")
-    return space.wrap(char)
+    return space.wrap(char) # YYY
 
 @unwrap_spec(code=int)
 def unichr(space, code):
diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/__init__.py
--- a/pypy/module/_collections/__init__.py
+++ b/pypy/module/_collections/__init__.py
@@ -21,5 +21,5 @@
         # been used...  otherwise, some code is not happy about seeing
         # this code object twice
         space = self.space
-        space.getattr(self, space.wrap('defaultdict'))  # force importing
-        space.delattr(self, space.wrap('__missing__'))
+        space.getattr(self, space.newtext('defaultdict'))  # force importing
+        space.delattr(self, space.newtext('__missing__'))
diff --git a/pypy/module/_collections/interp_defaultdict.py b/pypy/module/_collections/interp_defaultdict.py
--- a/pypy/module/_collections/interp_defaultdict.py
+++ b/pypy/module/_collections/interp_defaultdict.py
@@ -4,7 +4,7 @@
     # An interp-level version of this method.  This is mostly only
     # useful because it can be executed atomically in the presence of
     # threads.
-    w_default_factory = space.getattr(w_self, space.wrap('default_factory'))
+    w_default_factory = space.getattr(w_self, space.newtext('default_factory'))
     if space.is_w(w_default_factory, space.w_None):
         raise OperationError(space.w_KeyError, space.newtuple([w_key]))
     w_value = space.call_function(w_default_factory)
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -156,13 +156,13 @@
             if index >= BLOCKLEN:
                 block = block.rightlink
                 index = 0
-        return space.wrap(result)
+        return space.newint(result)
 
     def extend(self, w_iterable):
         "Extend the right side of the deque with elements from the iterable"
         # Handle case where id(deque) == id(iterable)
         space = self.space
-        if space.is_w(space.wrap(self), w_iterable):
+        if space.is_w(self, w_iterable):
             w_iterable = space.call_function(space.w_list, w_iterable)
         #
         w_iter = space.iter(w_iterable)
@@ -177,13 +177,13 @@
 
     def iadd(self, w_iterable):
         self.extend(w_iterable)
-        return self.space.wrap(self)
+        return self
 
     def extendleft(self, w_iterable):
         "Extend the left side of the deque with elements from the iterable"
         # Handle case where id(deque) == id(iterable)
         space = self.space
-        if space.is_w(space.wrap(self), w_iterable):
+        if space.is_w(self, w_iterable):
             w_iterable = space.call_function(space.w_list, w_iterable)
         #
         space = self.space
@@ -307,7 +307,7 @@
         return W_DequeRevIter(self)
 
     def length(self):
-        return self.space.wrap(self.len)
+        return self.space.newint(self.len)
 
     def repr(self):
         space = self.space
@@ -315,14 +315,14 @@
         w_currently_in_repr = ec._py_repr
         if w_currently_in_repr is None:
             w_currently_in_repr = ec._py_repr = space.newdict()
-        return dequerepr(space, w_currently_in_repr, space.wrap(self))
+        return dequerepr(space, w_currently_in_repr, self)
 
     @specialize.arg(2)
     def compare(self, w_other, op):
         space = self.space
         if not isinstance(w_other, W_Deque):
             return space.w_NotImplemented
-        return space.compare_by_iteration(space.wrap(self), w_other, op)
+        return space.compare_by_iteration(self, w_other, op)
 
     def lt(self, w_other):
         return self.compare(w_other, 'lt')
@@ -394,13 +394,13 @@
             return space.call_function(space.type(self), self)
         else:
             return space.call_function(space.type(self), self,
-                                       space.wrap(self.maxlen))
+                                       space.newint(self.maxlen))
 
     def reduce(self):
         "Return state information for pickling."
         space = self.space
         w_type = space.type(self)
-        w_dict = space.findattr(self, space.wrap('__dict__'))
+        w_dict = space.findattr(self, space.newtext('__dict__'))
         w_list = space.call_function(space.w_list, self)
         if w_dict is None:
             if self.maxlen == sys.maxint:
@@ -408,12 +408,12 @@
                     w_type, space.newtuple([w_list])]
             else:
                 result = [
-                    w_type, space.newtuple([w_list, space.wrap(self.maxlen)])]
+                    w_type, space.newtuple([w_list, space.newint(self.maxlen)])]
         else:
             if self.maxlen == sys.maxint:
                 w_len = space.w_None
             else:
-                w_len = space.wrap(self.maxlen)
+                w_len = space.newint(self.maxlen)
             result = [
                 w_type, space.newtuple([w_list, w_len]), w_dict]
         return space.newtuple(result)
@@ -422,7 +422,7 @@
         if self.maxlen == sys.maxint:
             return self.space.w_None
         else:
-            return self.space.wrap(self.maxlen)
+            return self.space.newint(self.maxlen)
 
 
 app = gateway.applevel("""
@@ -506,10 +506,10 @@
         check_nonneg(self.index)
 
     def iter(self):
-        return self.space.wrap(self)
+        return self
 
     def length(self):
-        return self.space.wrap(self.counter)
+        return self.space.newint(self.counter)
 
     def next(self):
         space = self.space
@@ -548,10 +548,10 @@
         check_nonneg(self.index)
 
     def iter(self):
-        return self.space.wrap(self)
+        return self
 
     def length(self):
-        return self.space.wrap(self.counter)
+        return self.space.newint(self.counter)
 
     def next(self):
         space = self.space
diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py
--- a/pypy/module/_file/interp_stream.py
+++ b/pypy/module/_file/interp_stream.py
@@ -114,13 +114,13 @@
                 result = self.stream.%(name)s(%(args)s)
             except streamio.StreamError, e:
                 raise OperationError(space.w_ValueError,
-                                     space.wrap(e.message))
+                                     space.newtext(e.message))
             except OSError, e:
                 raise wrap_oserror_as_ioerror(space, e)
         finally:
             if acquired:
                 self.release_lock()
-        return space.wrap(result)
+        return space.wrap(result) # YYY
     %(name)s.unwrap_spec = [W_Stream, ObjSpace] + argtypes
     """ % locals()).compile() in globals()
 
diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
--- a/pypy/module/_file/readinto.py
+++ b/pypy/module/_file/readinto.py
@@ -79,4 +79,4 @@
                     raise OSError(err, "read error")
             keepalive_until_here(rwbuffer)
 
-    return self.space.wrap(target_pos)
+    return self.space.newint(target_pos)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -142,14 +142,14 @@
             self.ll_chars[i+5] == 't' and
             self.ll_chars[i+6] == 'y'):
             self.pos = i+7
-            return self.space.wrap(rfloat.INFINITY * sign)
+            return self.space.newfloat(rfloat.INFINITY * sign)
         self._raise("Error when decoding Infinity at char %d", i)
 
     def decode_nan(self, i):
         if (self.ll_chars[i]   == 'a' and
             self.ll_chars[i+1] == 'N'):
             self.pos = i+2
-            return self.space.wrap(rfloat.NAN)
+            return self.space.newfloat(rfloat.NAN)
         self._raise("Error when decoding NaN at char %d", i)
 
     def decode_numeric(self, i):
@@ -168,7 +168,7 @@
             return self.decode_int_slow(start)
 
         self.pos = i
-        return self.space.wrap(intval)
+        return self.space.newint(intval)
 
     def decode_float(self, i):
         from rpython.rlib import rdtoa
@@ -176,7 +176,7 @@
         floatval = rdtoa.dg_strtod(start, self.end_ptr)
         diff = rffi.cast(rffi.LONG, self.end_ptr[0]) - rffi.cast(rffi.LONG, start)
         self.pos = i + diff
-        return self.space.wrap(floatval)
+        return self.space.newfloat(floatval)
 
     def decode_int_slow(self, i):
         start = i
@@ -186,7 +186,7 @@
             i += 1
         s = self.getslice(start, i)
         self.pos = i
-        return self.space.call_function(self.space.w_int, self.space.wrap(s))
+        return self.space.call_function(self.space.w_int, self.space.newtext(s))
 
     @always_inline
     def parse_integer(self, i):
@@ -306,7 +306,7 @@
                     content_unicode = strslice2unicode_latin1(self.s, start, i-1)
                 self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.wrap(content_unicode)
+                return self.space.newunicode(content_unicode)
             elif ch == '\\':
                 content_so_far = self.getslice(start, i-1)
                 self.pos = i-1
@@ -327,7 +327,7 @@
                 content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
                 self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.wrap(content_unicode)
+                return self.space.newunicode(content_unicode)
             elif ch == '\\':
                 i = self.decode_escape_sequence(i, builder)
             elif ch == '\0':
diff --git a/pypy/module/_sha/interp_sha.py b/pypy/module/_sha/interp_sha.py
--- a/pypy/module/_sha/interp_sha.py
+++ b/pypy/module/_sha/interp_sha.py
@@ -20,15 +20,15 @@
         self.update(string)
 
     def digest_w(self):
-        return self.space.wrap(self.digest())
+        return self.space.newbytes(self.digest())
 
     def hexdigest_w(self):
-        return self.space.wrap(self.hexdigest())
+        return self.space.newtext(self.hexdigest())
 
     def copy_w(self):
         clone = W_SHA(self.space)
         clone._copyfrom(self)
-        return self.space.wrap(clone)
+        return clone
 
 
 @unwrap_spec(initialdata='bufferstr')
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -277,9 +277,6 @@
             head = "closed"
         else:
             head = "open"
-        w_name = self.w_name
-        if w_name is None:
-            w_name = self.space.wrap('?')
         info = "%s bz2.BZ2File %s, mode '%s'" % (head, self.getdisplayname(),
                                                  self.mode)
         return self.getrepr(self.space, info)
@@ -287,7 +284,7 @@
 def descr_bz2file__new__(space, w_subtype, __args__):
     bz2file = space.allocate_instance(W_BZ2File, w_subtype)
     W_BZ2File.__init__(bz2file, space)
-    return space.wrap(bz2file)
+    return bz2file
 
 same_attributes_as_in_file = list(W_File._exposed_method_names)
 same_attributes_as_in_file.remove('__init__')
@@ -504,9 +501,8 @@
 @unwrap_spec(compresslevel=int)
 def descr_compressor__new__(space, w_subtype, compresslevel=9):
     x = space.allocate_instance(W_BZ2Compressor, w_subtype)
-    x = space.interp_w(W_BZ2Compressor, x)
     W_BZ2Compressor.__init__(x, space, compresslevel)
-    return space.wrap(x)
+    return x
 
 class W_BZ2Compressor(W_Root):
     """BZ2Compressor([compresslevel=9]) -> compressor object
@@ -614,9 +610,8 @@
 
 def descr_decompressor__new__(space, w_subtype):
     x = space.allocate_instance(W_BZ2Decompressor, w_subtype)
-    x = space.interp_w(W_BZ2Decompressor, x)
     W_BZ2Decompressor.__init__(x, space)
-    return space.wrap(x)
+    return x
 
 
 class W_BZ2Decompressor(W_Root):
@@ -750,7 +745,7 @@
 
                 res = out.make_result_string()
                 BZ2_bzCompressEnd(bzs)
-                return space.wrap(res)
+                return space.newbytes(res)
 
 @unwrap_spec(data='bufferstr')
 def decompress(space, data):
@@ -761,7 +756,7 @@
 
     in_bufsize = len(data)
     if in_bufsize == 0:
-        return space.wrap("")
+        return space.newbytes("")
 
     with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs:
         with rffi.scoped_nonmovingbuffer(data) as in_buf:
@@ -790,4 +785,4 @@
 
                 res = out.make_result_string()
                 BZ2_bzDecompressEnd(bzs)
-                return space.wrap(res)
+                return space.newbytes(res)

From pypy.commits at gmail.com  Wed Nov  2 09:03:37 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:37 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: math
Message-ID: <5819e429.41d71c0a.cca75.547a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88068:59437ad9ccb1
Date: 2016-11-02 12:08 +0100
http://bitbucket.org/pypy/pypy/changeset/59437ad9ccb1/

Log:	math

diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -7,8 +7,8 @@
 
 class State:
     def __init__(self, space):
-        self.w_e = space.wrap(math.e)
-        self.w_pi = space.wrap(math.pi)
+        self.w_e = space.newfloat(math.e)
+        self.w_pi = space.newfloat(math.pi)
 def get(space):
     return space.fromcache(State)
 
@@ -27,7 +27,7 @@
         raise oefmt(space.w_OverflowError, "math range error")
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
-    return space.wrap(y)
+    return space.newfloat(y)
 
 @specialize.arg(1)
 def math1_w(space, f, w_x):
@@ -50,7 +50,7 @@
         raise oefmt(space.w_OverflowError, "math range error")
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
-    return space.wrap(r)
+    return space.newfloat(r)
 
 def trunc(space, w_x):
     """Truncate x."""
@@ -61,15 +61,15 @@
     # No exceptions possible.
     x = _get_double(space, w_x)
     y = _get_double(space, w_y)
-    return space.wrap(rfloat.copysign(x, y))
+    return space.newfloat(rfloat.copysign(x, y))
 
 def isinf(space, w_x):
     """Return True if x is infinity."""
-    return space.wrap(rfloat.isinf(_get_double(space, w_x)))
+    return space.newbool(rfloat.isinf(_get_double(space, w_x)))
 
 def isnan(space, w_x):
     """Return True if x is not a number."""
-    return space.wrap(rfloat.isnan(_get_double(space, w_x)))
+    return space.newbool(rfloat.isnan(_get_double(space, w_x)))
 
 def pow(space, w_x, w_y):
     """pow(x,y)
@@ -96,7 +96,7 @@
         except OperationError as e:
             if not e.match(space, space.w_OverflowError):
                 raise
-            if space.is_true(space.lt(w_i, space.wrap(0))):
+            if space.is_true(space.lt(w_i, space.newint(0))):
                 exp = -sys.maxint
             else:
                 exp = sys.maxint
@@ -108,7 +108,7 @@
         raise oefmt(space.w_OverflowError, "math range error")
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
-    return space.wrap(r)
+    return space.newfloat(r)
 
 def hypot(space, w_x, w_y):
     """hypot(x,y)
@@ -145,7 +145,7 @@
        This is the largest integral value <= x.
     """
     x = _get_double(space, w_x)
-    return space.wrap(math.floor(x))
+    return space.newfloat(math.floor(x))
 
 def sqrt(space, w_x):
     """sqrt(x)
@@ -162,14 +162,14 @@
        If x is 0, m and e are both 0.  Else 0.5 <= abs(m) < 1.0.
     """
     mant, expo = math1_w(space, math.frexp, w_x)
-    return space.newtuple([space.wrap(mant), space.wrap(expo)])
+    return space.newtuple([space.newfloat(mant), space.newint(expo)])
 
 degToRad = math.pi / 180.0
 
 def degrees(space, w_x):
     """degrees(x) -> converts angle x from radians to degrees
     """
-    return space.wrap(_get_double(space, w_x) / degToRad)
+    return space.newfloat(_get_double(space, w_x) / degToRad)
 
 def _log_any(space, w_x, base):
     # base is supposed to be positive or 0.0, which means we use e
@@ -191,7 +191,7 @@
         raise oefmt(space.w_OverflowError, "math range error")
     except ValueError:
         raise oefmt(space.w_ValueError, "math domain error")
-    return space.wrap(result)
+    return space.newfloat(result)
 
 def log(space, w_x, w_base=None):
     """log(x[, base]) -> the logarithm of x to the given base.
@@ -257,7 +257,7 @@
 def radians(space, w_x):
     """radians(x) -> converts angle x from degrees to radians
     """
-    return space.wrap(_get_double(space, w_x) * degToRad)
+    return space.newfloat(_get_double(space, w_x) * degToRad)
 
 def sin(space, w_x):
     """sin(x)
@@ -281,7 +281,7 @@
        of x.  The integer part is returned as a real.
     """
     frac, intpart = math1_w(space, math.modf, w_x)
-    return space.newtuple([space.wrap(frac), space.wrap(intpart)])
+    return space.newtuple([space.newfloat(frac), space.newfloat(intpart)])
 
 def exp(space, w_x):
     """exp(x)
@@ -336,7 +336,7 @@
     if special_sum != 0.0:
         if rfloat.isnan(inf_sum):
             raise oefmt(space.w_ValueError, "-inf + inf")
-        return space.wrap(special_sum)
+        return space.newfloat(special_sum)
     hi = 0.0
     if partials:
         hi = partials[-1]
@@ -358,7 +358,7 @@
             yr = v - hi
             if y == yr:
                 hi = v
-    return space.wrap(hi)
+    return space.newfloat(hi)
 
 def log1p(space, w_x):
     """Find log(x + 1)."""

From pypy.commits at gmail.com  Wed Nov  2 09:03:39 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:39 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: __pypy__
Message-ID: <5819e42b.0777c20a.532be.d57e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88069:25160182f5b3
Date: 2016-11-02 12:38 +0100
http://bitbucket.org/pypy/pypy/changeset/25160182f5b3/

Log:	__pypy__

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -6,7 +6,7 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 
-def create_builder(name, strtype, builder_cls):
+def create_builder(name, strtype, builder_cls, newmethod):
     class W_Builder(W_Root):
         def __init__(self, space, size):
             if size < 0:
@@ -37,14 +37,14 @@
 
         def descr_build(self, space):
             self._check_done(space)
-            w_s = space.wrap(self.builder.build())
+            w_s = getattr(space, newmethod)(self.builder.build())
             self.builder = None
             return w_s
 
         def descr_len(self, space):
             if self.builder is None:
                 raise oefmt(space.w_ValueError, "no length of built builder")
-            return space.wrap(self.builder.getlength())
+            return space.newint(self.builder.getlength())
 
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
@@ -59,5 +59,5 @@
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
 
-W_StringBuilder = create_builder("StringBuilder", str, StringBuilder)
-W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder)
+W_StringBuilder = create_builder("StringBuilder", str, StringBuilder, "newbytes")
+W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder, "newunicode")
diff --git a/pypy/module/__pypy__/interp_identitydict.py b/pypy/module/__pypy__/interp_identitydict.py
--- a/pypy/module/__pypy__/interp_identitydict.py
+++ b/pypy/module/__pypy__/interp_identitydict.py
@@ -11,13 +11,13 @@
     def descr_new(space, w_subtype):
         self = space.allocate_instance(W_IdentityDict, w_subtype)
         W_IdentityDict.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     def descr_len(self, space):
-        return space.wrap(len(self.dict))
+        return space.newint(len(self.dict))
 
     def descr_contains(self, space, w_key):
-        return space.wrap(w_key in self.dict)
+        return space.newbool(w_key in self.dict)
 
     def descr_setitem(self, space, w_key, w_value):
         self.dict[w_key] = w_value
diff --git a/pypy/module/__pypy__/interp_intop.py b/pypy/module/__pypy__/interp_intop.py
--- a/pypy/module/__pypy__/interp_intop.py
+++ b/pypy/module/__pypy__/interp_intop.py
@@ -8,34 +8,34 @@
 
 @unwrap_spec(n=int, m=int)
 def int_add(space, n, m):
-    return space.wrap(llop.int_add(lltype.Signed, n, m))
+    return space.newint(llop.int_add(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_sub(space, n, m):
-    return space.wrap(llop.int_sub(lltype.Signed, n, m))
+    return space.newint(llop.int_sub(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mul(space, n, m):
-    return space.wrap(llop.int_mul(lltype.Signed, n, m))
+    return space.newint(llop.int_mul(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_floordiv(space, n, m):
-    return space.wrap(int_c_div(n, m))
+    return space.newint(int_c_div(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_mod(space, n, m):
-    return space.wrap(int_c_mod(n, m))
+    return space.newint(int_c_mod(n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_lshift(space, n, m):
-    return space.wrap(llop.int_lshift(lltype.Signed, n, m))
+    return space.newint(llop.int_lshift(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def int_rshift(space, n, m):
-    return space.wrap(llop.int_rshift(lltype.Signed, n, m))
+    return space.newint(llop.int_rshift(lltype.Signed, n, m))
 
 @unwrap_spec(n=int, m=int)
 def uint_rshift(space, n, m):
     n = r_uint(n)
     x = llop.uint_rshift(lltype.Unsigned, n, m)
-    return space.wrap(intmask(x))
+    return space.newint(intmask(x))
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -13,7 +13,7 @@
 
 
 def internal_repr(space, w_object):
-    return space.wrap('%r' % (w_object,))
+    return space.newtext('%r' % (w_object,))
 
 
 def attach_gdb(space):
@@ -59,7 +59,7 @@
     from pypy.interpreter.function import Function, BuiltinFunction
     func = space.interp_w(Function, w_func)
     bltn = BuiltinFunction(func)
-    return space.wrap(bltn)
+    return bltn
 
 def hidden_applevel(space, w_func):
     """Decorator that hides a function's frame from app-level"""
@@ -73,7 +73,7 @@
     frame hidden from applevel.
     """
     operr = space.getexecutioncontext().sys_exc_info(for_hidden=True)
-    return space.w_None if operr is None else space.wrap(operr.get_traceback())
+    return space.w_None if operr is None else operr.get_traceback()
 
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
@@ -87,7 +87,7 @@
     return space.get(w_descr, w_obj)
 
 def do_what_I_mean(space):
-    return space.wrap(42)
+    return space.newint(42)
 
 
 def strategy(space, w_obj):
@@ -103,7 +103,7 @@
         name = w_obj.strategy.__class__.__name__
     else:
         raise oefmt(space.w_TypeError, "expecting dict or list or set object")
-    return space.wrap(name)
+    return space.newtext(name)
 
 
 @unwrap_spec(fd='c_int')
@@ -116,8 +116,8 @@
 def get_console_cp(space):
     from rpython.rlib import rwin32    # Windows only
     return space.newtuple([
-        space.wrap('cp%d' % rwin32.GetConsoleCP()),
-        space.wrap('cp%d' % rwin32.GetConsoleOutputCP()),
+        space.newtext('cp%d' % rwin32.GetConsoleCP()),
+        space.newtext('cp%d' % rwin32.GetConsoleOutputCP()),
         ])
 
 @unwrap_spec(sizehint=int)
@@ -134,8 +134,8 @@
 def set_debug(space, debug):
     space.sys.debug = debug
     space.setitem(space.builtin.w_dict,
-                  space.wrap('__debug__'),
-                  space.wrap(debug))
+                  space.newtext('__debug__'),
+                  space.newbool(debug))
 
 @unwrap_spec(estimate=int)
 def add_memory_pressure(estimate):
diff --git a/pypy/module/__pypy__/interp_os.py b/pypy/module/__pypy__/interp_os.py
--- a/pypy/module/__pypy__/interp_os.py
+++ b/pypy/module/__pypy__/interp_os.py
@@ -6,4 +6,4 @@
 @unwrap_spec(name='str0')
 def real_getenv(space, name):
     """Get an OS environment value skipping Python cache"""
-    return space.wrap(os.environ.get(name))
+    return space.newtext_or_none(os.environ.get(name))
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -18,7 +18,7 @@
                 raise exception_from_saved_errno(space, space.w_IOError)
             t = (float(rffi.getintfield(tp, 'c_tv_sec')) +
                  float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001)
-        return space.wrap(t)
+        return space.newfloat(t)
 
     @unwrap_spec(clk_id="c_int")
     def clock_getres(space, clk_id):
@@ -28,4 +28,4 @@
                 raise exception_from_saved_errno(space, space.w_IOError)
             t = (float(rffi.getintfield(tp, 'c_tv_sec')) +
                  float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001)
-        return space.wrap(t)
+        return space.newfloat(t)
diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py
--- a/pypy/module/binascii/interp_hqx.py
+++ b/pypy/module/binascii/interp_hqx.py
@@ -97,7 +97,7 @@
     else:
         if pending_bits > 0:
             raise_Incomplete(space, 'String has incomplete number of bytes')
-    return space.newtuple([space.newbytes(res.build()), space.wrap(done)])
+    return space.newtuple([space.newbytes(res.build()), space.newint(done)])
 
 # ____________________________________________________________
 
@@ -242,4 +242,4 @@
     crc = oldcrc
     for c in data:
         crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
-    return space.wrap(crc)
+    return space.newint(crc)

From pypy.commits at gmail.com  Wed Nov  2 09:03:42 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:42 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: cmath
Message-ID: <5819e42e.49a1c20a.6333f.d410@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88071:932f15c35f9d
Date: 2016-11-02 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/932f15c35f9d/

Log:	cmath

diff --git a/pypy/module/cmath/__init__.py b/pypy/module/cmath/__init__.py
--- a/pypy/module/cmath/__init__.py
+++ b/pypy/module/cmath/__init__.py
@@ -37,8 +37,8 @@
     }
 
     interpleveldefs = {
-        'pi': 'space.wrap(interp_cmath.pi)',
-        'e':  'space.wrap(interp_cmath.e)',
+        'pi': 'space.newfloat(interp_cmath.pi)',
+        'e':  'space.newfloat(interp_cmath.e)',
     }
     interpleveldefs.update(dict([(name, 'interp_cmath.wrapped_' + name)
                                  for name in names_and_docstrings]))

From pypy.commits at gmail.com  Wed Nov  2 09:03:44 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:44 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: operator
Message-ID: <5819e430.11881c0a.b09d1.5c18@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88072:3e71c512f034
Date: 2016-11-02 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/3e71c512f034/

Log:	operator

diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -245,13 +245,13 @@
 
 @unwrap_spec(default=int)
 def _length_hint(space, w_iterable, default):
-    return space.wrap(space.length_hint(w_iterable, default))
+    return space.newint(space.length_hint(w_iterable, default))
 
 
 def isMappingType(space, w_obj):
     'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
-    return space.wrap(space.ismapping_w(w_obj))
+    return space.newbool(space.ismapping_w(w_obj))
 
 def isSequenceType(space, w_obj):
     'isSequenceType(a) -- Return True if a has a sequence type, False otherwise.'
-    return space.wrap(space.issequence_w(w_obj))
+    return space.newbool(space.issequence_w(w_obj))
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
--- a/pypy/module/operator/tscmp.py
+++ b/pypy/module/operator/tscmp.py
@@ -53,7 +53,7 @@
         with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
             with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
                 result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
-        return space.wrap(rffi.cast(lltype.Bool, result))
+        return space.newbool(rffi.cast(lltype.Bool, result))
     return compare_digest_buffer(space, w_a, w_b)
 
 
@@ -71,4 +71,4 @@
     with rffi.scoped_nonmovingbuffer(a) as a_buf:
         with rffi.scoped_nonmovingbuffer(b) as b_buf:
             result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
-    return space.wrap(rffi.cast(lltype.Bool, result))
+    return space.newbool(rffi.cast(lltype.Bool, result))

From pypy.commits at gmail.com  Wed Nov  2 09:03:40 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:40 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: exceptions
Message-ID: <5819e42c.4a6ec20a.b8d27.d517@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88070:0e81f92dc7cb
Date: 2016-11-02 13:06 +0100
http://bitbucket.org/pypy/pypy/changeset/0e81f92dc7cb/

Log:	exceptions

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -106,12 +106,12 @@
         if len(args_w) == 1:
             self.w_message = args_w[0]
         else:
-            self.w_message = space.wrap("")
+            self.w_message = space.newtext("")
 
     def descr_str(self, space):
         lgt = len(self.args_w)
         if lgt == 0:
-            return space.wrap('')
+            return space.newtext('')
         elif lgt == 1:
             return space.str(self.args_w[0])
         else:
@@ -122,11 +122,11 @@
         w_base_type = space.gettypeobject(W_BaseException.typedef)
         w_base_str = w_base_type.dict_w["__str__"]
         if not space.is_w(w_str, w_base_str):
-            w_as_str = space.get_and_call_function(w_str, space.wrap(self))
+            w_as_str = space.get_and_call_function(w_str, self)
             return space.call_function(space.w_unicode, w_as_str)
         lgt = len(self.args_w)
         if lgt == 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
         if lgt == 1:
             return space.call_function(space.w_unicode, self.args_w[0])
         else:
@@ -139,7 +139,7 @@
         else:
             args_repr = "()"
         clsname = self.getclass(space).getname(space)
-        return space.wrap(clsname + args_repr)
+        return space.newtext(clsname + args_repr)
 
     def descr_getargs(self, space):
         return space.newtuple(self.args_w)
@@ -174,23 +174,23 @@
     def descr_message_get(self, space):
         w_dict = self.w_dict
         if w_dict is not None:
-            w_msg = space.finditem(w_dict, space.wrap("message"))
+            w_msg = space.finditem(w_dict, space.newtext("message"))
             if w_msg is not None:
                 return w_msg
         if self.w_message is None:
             raise oefmt(space.w_AttributeError, "message was deleted")
         msg = "BaseException.message has been deprecated as of Python 2.6"
-        space.warn(space.wrap(msg), space.w_DeprecationWarning)
+        space.warn(space.newtext(msg), space.w_DeprecationWarning)
         return self.w_message
 
     def descr_message_set(self, space, w_new):
-        space.setitem(self.getdict(space), space.wrap("message"), w_new)
+        space.setitem(self.getdict(space), space.newtext("message"), w_new)
 
     def descr_message_del(self, space):
         w_dict = self.w_dict
         if w_dict is not None:
             try:
-                space.delitem(w_dict, space.wrap("message"))
+                space.delitem(w_dict, space.newtext("message"))
             except OperationError as e:
                 if not e.match(space, space.w_KeyError):
                     raise
@@ -202,7 +202,7 @@
     def descr_new_base_exception(space, w_subtype, __args__):
         exc = space.allocate_instance(cls, w_subtype)
         basecls.__init__(exc, space)
-        return space.wrap(exc)
+        return exc
     descr_new_base_exception.func_name = 'descr_new_' + cls.__name__
     return interp2app(descr_new_base_exception)
 
@@ -298,7 +298,7 @@
                                                  w_end, w_reason])
 
     def descr_str(self, space):
-        return space.appexec([space.wrap(self)], r"""(self):
+        return space.appexec([self], r"""(self):
             if self.object is None:
                 return ""
             if self.end == self.start + 1:
@@ -329,7 +329,7 @@
 
 def key_error_str(self, space):
     if len(self.args_w) == 0:
-        return space.wrap('')
+        return space.newtext('')
     elif len(self.args_w) == 1:
         return space.repr(self.args_w[0])
     else:
@@ -384,11 +384,11 @@
             errno = space.str_w(space.str(self.w_errno))
             strerror = space.str_w(space.str(self.w_strerror))
             if not space.is_w(self.w_filename, space.w_None):
-                return space.wrap("[Errno %s] %s: %s" % (
+                return space.newtext("[Errno %s] %s: %s" % (
                     errno,
                     strerror,
                     space.str_w(space.repr(self.w_filename))))
-            return space.wrap("[Errno %s] %s" % (
+            return space.newtext("[Errno %s] %s" % (
                 errno,
                 strerror,
             ))
@@ -428,18 +428,18 @@
         else:
             errno = self._winerror_to_errno.get(errno, self._default_errno)
         self.w_winerror = self.w_errno
-        self.w_errno = space.wrap(errno)
+        self.w_errno = space.newint(errno)
 
     def descr_str(self, space):
         if (not space.is_w(self.w_winerror, space.w_None) and
             not space.is_w(self.w_strerror, space.w_None)):
             if not space.is_w(self.w_filename, space.w_None):
-                return space.wrap("[Error %d] %s: %s" % (
+                return space.newtext("[Error %d] %s: %s" % (
                     space.int_w(self.w_winerror),
                     space.str_w(self.w_strerror),
                     space.str_w(self.w_filename)))
-            return space.wrap("[Error %d] %s" % (space.int_w(self.w_winerror),
-                                                 space.str_w(self.w_strerror)))
+            return space.newtext("[Error %d] %s" % (space.int_w(self.w_winerror),
+                                                    space.str_w(self.w_strerror)))
         return W_BaseException.descr_str(self, space)
 
     if hasattr(rwin32, 'build_winerror_to_errno'):
@@ -549,7 +549,7 @@
             args_w = [self.args_w[0], w_tuple]
             args_repr = space.str_w(space.repr(space.newtuple(args_w)))
             clsname = self.getclass(space).getname(space)
-            return space.wrap(clsname + args_repr)
+            return space.newtext(clsname + args_repr)
         else:
             return W_StandardError.descr_repr(self, space)
 

From pypy.commits at gmail.com  Wed Nov  2 09:03:45 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:45 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: select
Message-ID: <5819e431.41d71c0a.cca75.5495@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88073:8604f987eddd
Date: 2016-11-02 13:08 +0100
http://bitbucket.org/pypy/pypy/changeset/8604f987eddd/

Log:	select

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -681,7 +681,7 @@
         return self.sig
 
     def getdocstring(self, space):
-        return space.newtext(self.docstring)
+        return space.newtext_or_none(self.docstring)
 
     def funcrun(self, func, args):
         return BuiltinCode.funcrun_obj(self, func, None, args)
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -93,11 +93,11 @@
         if epfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
 
-        return space.wrap(W_Epoll(space, epfd))
+        return W_Epoll(space, epfd)
 
     @unwrap_spec(fd=int)
     def descr_fromfd(space, w_cls, fd):
-        return space.wrap(W_Epoll(space, fd))
+        return W_Epoll(space, fd)
 
     def _finalize_(self):
         self.close()
@@ -127,11 +127,11 @@
                 raise exception_from_saved_errno(space, space.w_IOError)
 
     def descr_get_closed(self, space):
-        return space.wrap(self.get_closed())
+        return space.newbool(self.get_closed())
 
     def descr_fileno(self, space):
         self.check_closed(space)
-        return space.wrap(self.epfd)
+        return space.newint(self.epfd)
 
     def descr_close(self, space):
         self.close()
@@ -173,7 +173,7 @@
             for i in xrange(nfds):
                 event = evs[i]
                 elist_w[i] = space.newtuple(
-                    [space.wrap(event.c_data.c_fd), space.wrap(event.c_events)]
+                    [space.newint(event.c_data.c_fd), space.newint(event.c_events)]
                 )
             return space.newlist(elist_w)
 
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -115,11 +115,11 @@
         kqfd = syscall_kqueue()
         if kqfd < 0:
             raise exception_from_saved_errno(space, space.w_IOError)
-        return space.wrap(W_Kqueue(space, kqfd))
+        return W_Kqueue(space, kqfd)
 
     @unwrap_spec(fd=int)
     def descr_fromfd(space, w_cls, fd):
-        return space.wrap(W_Kqueue(space, fd))
+        return W_Kqueue(space, fd)
 
     def _finalize_(self):
         self.close()
@@ -139,11 +139,11 @@
                         "I/O operation on closed kqueue fd")
 
     def descr_get_closed(self, space):
-        return space.wrap(self.get_closed())
+        return space.newbool(self.get_closed())
 
     def descr_fileno(self, space):
         self.check_closed(space)
-        return space.wrap(self.kqfd)
+        return space.newint(self.kqfd)
 
     def descr_close(self, space):
         self.close()
@@ -317,40 +317,40 @@
         return self._compare_all_fields(space.interp_w(W_Kevent, other), op)
 
     def descr__eq__(self, space, w_other):
-        return space.wrap(self.compare_all_fields(space, w_other, "eq"))
+        return space.newbool(self.compare_all_fields(space, w_other, "eq"))
 
     def descr__ne__(self, space, w_other):
-        return space.wrap(not self.compare_all_fields(space, w_other, "eq"))
+        return space.newbool(not self.compare_all_fields(space, w_other, "eq"))
 
     def descr__le__(self, space, w_other):
-        return space.wrap(not self.compare_all_fields(space, w_other, "gt"))
+        return space.newbool(not self.compare_all_fields(space, w_other, "gt"))
 
     def descr__lt__(self, space, w_other):
-        return space.wrap(self.compare_all_fields(space, w_other, "lt"))
+        return space.newbool(self.compare_all_fields(space, w_other, "lt"))
 
     def descr__ge__(self, space, w_other):
-        return space.wrap(not self.compare_all_fields(space, w_other, "lt"))
+        return space.newbool(not self.compare_all_fields(space, w_other, "lt"))
 
     def descr__gt__(self, space, w_other):
-        return space.wrap(self.compare_all_fields(space, w_other, "gt"))
+        return space.newbool(self.compare_all_fields(space, w_other, "gt"))
 
     def descr_get_ident(self, space):
-        return space.wrap(self.ident)
+        return space.newint(self.ident)
 
     def descr_get_filter(self, space):
-        return space.wrap(self.filter)
+        return space.newint(self.filter)
 
     def descr_get_flags(self, space):
-        return space.wrap(self.flags)
+        return space.newint(self.flags)
 
     def descr_get_fflags(self, space):
-        return space.wrap(self.fflags)
+        return space.newint(self.fflags)
 
     def descr_get_data(self, space):
-        return space.wrap(self.data)
+        return space.newint(self.data)
 
     def descr_get_udata(self, space):
-        return space.wrap(rffi.cast(rffi.UINTPTR_T, self.udata))
+        return space.newint(rffi.cast(rffi.UINTPTR_T, self.udata))
 
 
 W_Kevent.typedef = TypeDef("select.kevent",
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -48,7 +48,7 @@
         try:
             del self.fddict[fd]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap(fd))
+            raise OperationError(space.w_KeyError, space.newint(fd))
 
     @unwrap_spec(w_timeout=WrappedDefault(None))
     def poll(self, space, w_timeout):
@@ -74,15 +74,15 @@
             w_errortype = space.fromcache(Cache).w_error
             message = e.get_msg()
             raise OperationError(w_errortype,
-                                 space.newtuple([space.wrap(e.errno),
-                                                 space.wrap(message)]))
+                                 space.newtuple([space.newint(e.errno),
+                                                 space.newtext(message)]))
         finally:
             self.running = False
 
         retval_w = []
         for fd, revents in retval:
-            retval_w.append(space.newtuple([space.wrap(fd),
-                                            space.wrap(revents)]))
+            retval_w.append(space.newtuple([space.newint(fd),
+                                            space.newtext(revents)]))
         return space.newlist(retval_w)
 
 pollmethods = {}
@@ -133,7 +133,7 @@
         msg = _c.socket_strerror_str(errno)
         w_errortype = space.fromcache(Cache).w_error
         raise OperationError(w_errortype, space.newtuple([
-            space.wrap(errno), space.wrap(msg)]))
+            space.newint(errno), space.newtext(msg)]))
 
     resin_w = []
     resout_w = []

From pypy.commits at gmail.com  Wed Nov  2 09:03:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:47 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: zlib
Message-ID: <5819e433.120f1c0a.748ba.efee@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88074:75357e535417
Date: 2016-11-02 13:10 +0100
http://bitbucket.org/pypy/pypy/changeset/75357e535417/

Log:	zlib

diff --git a/pypy/module/zlib/__init__.py b/pypy/module/zlib/__init__.py
--- a/pypy/module/zlib/__init__.py
+++ b/pypy/module/zlib/__init__.py
@@ -30,7 +30,7 @@
         'decompressobj': 'interp_zlib.Decompress',
         'compress': 'interp_zlib.compress',
         'decompress': 'interp_zlib.decompress',
-        '__version__': 'space.wrap("1.0")',
+        '__version__': 'space.newtext("1.0")',
         'error': 'space.fromcache(interp_zlib.Cache).w_error',
         }
 
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -38,7 +38,7 @@
     # the 2.6 behavior and always return a number in range(-2**31, 2**31).
     checksum = unsigned_to_signed_32bit(checksum)
 
-    return space.wrap(checksum)
+    return space.newint(checksum)
 
 
 @unwrap_spec(string='bufferstr', start='truncatedint_w')
@@ -54,7 +54,7 @@
     # See comments in crc32() for the following line
     checksum = unsigned_to_signed_32bit(checksum)
 
-    return space.wrap(checksum)
+    return space.newint(checksum)
 
 
 class Cache:
@@ -63,7 +63,7 @@
 
 def zlib_error(space, msg):
     w_error = space.fromcache(Cache).w_error
-    return OperationError(w_error, space.wrap(msg))
+    return OperationError(w_error, space.newtext(msg))
 
 
 @unwrap_spec(string='bufferstr', level=int)
@@ -84,7 +84,7 @@
             rzlib.deflateEnd(stream)
     except rzlib.RZlibError as e:
         raise zlib_error(space, e.msg)
-    return space.wrap(result)
+    return space.newbytes(result)
 
 
 @unwrap_spec(string='bufferstr', wbits="c_int", bufsize=int)
@@ -106,7 +106,7 @@
             rzlib.inflateEnd(stream)
     except rzlib.RZlibError as e:
         raise zlib_error(space, e.msg)
-    return space.wrap(result)
+    return space.newbytes(result)
 
 
 class ZLibObject(W_Root):
@@ -177,7 +177,7 @@
                 self.unlock()
         except rzlib.RZlibError as e:
             raise zlib_error(space, e.msg)
-        return space.wrap(result)
+        return space.newbytes(result)
 
     @unwrap_spec(mode="c_int")
     def flush(self, space, mode=rzlib.Z_FINISH):
@@ -206,7 +206,7 @@
                 self.unlock()
         except rzlib.RZlibError as e:
             raise zlib_error(space, e.msg)
-        return space.wrap(result)
+        return space.newbytes(result)
 
 
 @unwrap_spec(level=int, method=int, wbits=int, memLevel=int, strategy=int)
@@ -222,7 +222,7 @@
     stream = space.interp_w(Compress, stream)
     Compress.__init__(stream, space, level,
                       method, wbits, memLevel, strategy)
-    return space.wrap(stream)
+    return stream
 
 
 Compress.typedef = TypeDef(
@@ -303,7 +303,7 @@
 
         string, finished, unused_len = result
         self._save_unconsumed_input(data, finished, unused_len)
-        return space.wrap(string)
+        return space.newbytes(string)
 
     def flush(self, space, w_length=None):
         """
@@ -328,7 +328,7 @@
         else:
             string, finished, unused_len = result
             self._save_unconsumed_input(data, finished, unused_len)
-        return space.wrap(string)
+        return space.newbytes(string)
 
 
 @unwrap_spec(wbits=int)
@@ -339,7 +339,7 @@
     stream = space.allocate_instance(Decompress, w_subtype)
     stream = space.interp_w(Decompress, stream)
     Decompress.__init__(stream, space, wbits)
-    return space.wrap(stream)
+    return stream
 
 
 Decompress.typedef = TypeDef(

From pypy.commits at gmail.com  Wed Nov  2 09:03:49 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:49 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _codecs, _multibytecodec
Message-ID: <5819e435.49151c0a.2531f.63a3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88075:dd3d24ec655b
Date: 2016-11-02 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/dd3d24ec655b/

Log:	_codecs, _multibytecodec

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -42,16 +42,16 @@
                 w_input = space.newunicode(input)
             w_exc =  space.call_function(
                 w_cls,
-                space.wrap(encoding),
+                space.newtext(encoding),
                 w_input,
-                space.wrap(startpos),
-                space.wrap(endpos),
-                space.wrap(reason))
+                space.newint(startpos),
+                space.newint(endpos),
+                space.newtext(reason))
             w_res = space.call_function(w_errorhandler, w_exc)
             if (not space.isinstance_w(w_res, space.w_tuple)
                 or space.len_w(w_res) != 2
                 or not space.isinstance_w(
-                                 space.getitem(w_res, space.wrap(0)),
+                                 space.getitem(w_res, space.newint(0)),
                                  space.w_unicode)):
                 raise oefmt(space.w_TypeError,
                             "%s error handler must return (unicode, int) "
@@ -86,7 +86,7 @@
             return self.unicodedata_handler
         try:
             w_unicodedata = space.getbuiltinmodule("unicodedata")
-            w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code"))
+            w_getcode = space.getattr(w_unicodedata, space.newtext("_get_code"))
         except OperationError:
             return None
         else:
@@ -140,9 +140,9 @@
 def _lookup_codec_loop(space, encoding, normalized_encoding):
     state = space.fromcache(CodecState)
     if state.codec_need_encodings:
-        w_import = space.getattr(space.builtin, space.wrap("__import__"))
+        w_import = space.getattr(space.builtin, space.newtext("__import__"))
         # registers new codecs
-        space.call_function(w_import, space.wrap("encodings"))
+        space.call_function(w_import, space.newtext("encodings"))
         state.codec_need_encodings = False
         if len(state.codec_search_path) == 0:
             raise oefmt(space.w_LookupError,
@@ -150,7 +150,7 @@
                         "encoding")
     for w_search in state.codec_search_path:
         w_result = space.call_function(w_search,
-                                       space.wrap(normalized_encoding))
+                                       space.newtext(normalized_encoding))
         if not space.is_w(w_result, space.w_None):
             if not (space.isinstance_w(w_result, space.w_tuple) and
                     space.len_w(w_result) == 4):
@@ -167,9 +167,9 @@
 
 def check_exception(space, w_exc):
     try:
-        w_start = space.getattr(w_exc, space.wrap('start'))
-        w_end = space.getattr(w_exc, space.wrap('end'))
-        w_obj = space.getattr(w_exc, space.wrap('object'))
+        w_start = space.getattr(w_exc, space.newtext('start'))
+        w_end = space.getattr(w_exc, space.newtext('end'))
+        w_obj = space.getattr(w_exc, space.newtext('object'))
     except OperationError as e:
         if not e.match(space, space.w_AttributeError):
             raise
@@ -189,23 +189,23 @@
 
 def ignore_errors(space, w_exc):
     check_exception(space, w_exc)
-    w_end = space.getattr(w_exc, space.wrap('end'))
-    return space.newtuple([space.wrap(u''), w_end])
+    w_end = space.getattr(w_exc, space.newtext('end'))
+    return space.newtuple([space.newunicode(u''), w_end])
 
 def replace_errors(space, w_exc):
     check_exception(space, w_exc)
-    w_start = space.getattr(w_exc, space.wrap('start'))
-    w_end = space.getattr(w_exc, space.wrap('end'))
+    w_start = space.getattr(w_exc, space.newtext('start'))
+    w_end = space.getattr(w_exc, space.newtext('end'))
     size = space.int_w(w_end) - space.int_w(w_start)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
         text = u'?' * size
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         text = u'\ufffd'
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeTranslateError):
         text = u'\ufffd' * size
-        return space.newtuple([space.wrap(text), w_end])
+        return space.newtuple([space.newunicode(text), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -213,9 +213,9 @@
 def xmlcharrefreplace_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -231,7 +231,7 @@
             builder.append(unicode(str(code)))
             builder.append(u";")
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -239,9 +239,9 @@
 def backslashreplace_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
-        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
-        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
-        w_end = space.getattr(w_exc, space.wrap('end'))
+        obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object')))
+        start = space.int_w(space.getattr(w_exc, space.newtext('start')))
+        w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
         builder = UnicodeBuilder()
         pos = start
@@ -263,7 +263,7 @@
                 builder.append_multiple_char(u'0', nb)
             builder.append_slice(unicode(num), 2, lnum)
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newunicode(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -309,19 +309,19 @@
         encoding = space.sys.defaultencoding
     else:
         encoding = space.str_w(w_encoding)
-    w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
-    w_res = space.call_function(w_encoder, w_obj, space.wrap(errors))
-    return space.getitem(w_res, space.wrap(0))
+    w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
+    w_res = space.call_function(w_encoder, w_obj, space.newtext(errors))
+    return space.getitem(w_res, space.newint(0))
 
 @unwrap_spec(errors='str_or_None')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
-    return space.newtuple([space.newbytes(s), space.wrap(len(s))])
+    return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
 @unwrap_spec(errors='str_or_None')
 def charbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('t#', w_data)
-    return space.newtuple([space.wrap(s), space.wrap(len(s))])
+    return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
 @unwrap_spec(errors=str)
 def decode(space, w_obj, w_encoding=None, errors='strict'):
@@ -338,13 +338,13 @@
         encoding = space.sys.defaultencoding
     else:
         encoding = space.str_w(w_encoding)
-    w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1))
+    w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
     if space.is_true(w_decoder):
-        w_res = space.call_function(w_decoder, w_obj, space.wrap(errors))
+        w_res = space.call_function(w_decoder, w_obj, space.newtext(errors))
         if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
             raise oefmt(space.w_TypeError,
                         "encoder must return a tuple (object, integer)")
-        return space.getitem(w_res, space.wrap(0))
+        return space.getitem(w_res, space.newint(0))
     else:
         assert 0, "XXX, what to do here?"
 
@@ -379,7 +379,7 @@
         state = space.fromcache(CodecState)
         func = getattr(runicode, rname)
         result = func(uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+        return space.newtuple([space.newbytes(result), space.newint(len(uni))])
     wrap_encoder.func_name = rname
     globals()[name] = wrap_encoder
 
@@ -396,7 +396,7 @@
         func = getattr(runicode, rname)
         result, consumed = func(string, len(string), errors,
                                 final, state.decode_error_handler)
-        return space.newtuple([space.wrap(result), space.wrap(consumed)])
+        return space.newtuple([space.newunicode(result), space.newint(consumed)])
     wrap_decoder.func_name = rname
     globals()[name] = wrap_decoder
 
@@ -444,7 +444,7 @@
     result = runicode.unicode_encode_utf_8(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=True)
-    return space.newtuple([space.wrap(result), space.wrap(len(uni))])
+    return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 @unwrap_spec(string='bufferstr', errors='str_or_None',
              w_final = WrappedDefault(False))
@@ -457,7 +457,7 @@
         string, len(string), errors,
         final, state.decode_error_handler,
         allow_surrogates=True)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -477,8 +477,8 @@
         consumed = 0
     res, consumed, byteorder = runicode.str_decode_utf_16_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
-    return space.newtuple([space.wrap(res), space.wrap(consumed),
-                           space.wrap(byteorder)])
+    return space.newtuple([space.newunicode(res), space.newint(consumed),
+                           space.newtext(byteorder)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -496,8 +496,8 @@
         consumed = 0
     res, consumed, byteorder = runicode.str_decode_utf_32_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
-    return space.newtuple([space.wrap(res), space.wrap(consumed),
-                           space.wrap(byteorder)])
+    return space.newtuple([space.newunicode(res), space.newint(consumed),
+                           space.newtext(byteorder)])
 
 # ____________________________________________________________
 # Charmap
@@ -583,7 +583,7 @@
     if errors is None:
         errors = 'strict'
     if len(string) == 0:
-        return space.newtuple([space.wrap(u''), space.wrap(0)])
+        return space.newtuple([space.newunicode(u''), space.newint(0)])
 
     if space.is_none(w_mapping):
         mapping = None
@@ -595,7 +595,7 @@
     result, consumed = runicode.str_decode_charmap(
         string, len(string), errors,
         final, state.decode_error_handler, mapping)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 @unwrap_spec(uni=unicode, errors='str_or_None')
 def charmap_encode(space, uni, errors="strict", w_mapping=None):
@@ -610,7 +610,7 @@
     result = runicode.unicode_encode_charmap(
         uni, len(uni), errors,
         state.encode_error_handler, mapping)
-    return space.newtuple([space.newbytes(result), space.wrap(len(uni))])
+    return space.newtuple([space.newbytes(result), space.newint(len(uni))])
 
 
 @unwrap_spec(chars=unicode)
@@ -633,7 +633,7 @@
     def call(self, name):
         space = self.space
         try:
-            w_code = space.call_function(self.w_getcode, space.wrap(name))
+            w_code = space.call_function(self.w_getcode, space.newtext(name))
         except OperationError as e:
             if not e.match(space, space.w_KeyError):
                 raise
@@ -655,7 +655,7 @@
         final, state.decode_error_handler,
         unicode_name_handler)
 
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
 # Unicode-internal
@@ -671,14 +671,14 @@
     string = space.readbuf_w(w_string).as_str()
 
     if len(string) == 0:
-        return space.newtuple([space.wrap(u''), space.wrap(0)])
+        return space.newtuple([space.newunicode(u''), space.newint(0)])
 
     final = True
     state = space.fromcache(CodecState)
     result, consumed = runicode.str_decode_unicode_internal(
         string, len(string), errors,
         final, state.decode_error_handler)
-    return space.newtuple([space.wrap(result), space.wrap(consumed)])
+    return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
 # support for the "string escape" codec
@@ -691,11 +691,11 @@
     start = 1
     end = len(result) - 1
     assert end >= 0
-    w_result = space.wrap(result[start:end])
-    return space.newtuple([w_result, space.wrap(len(data))])
+    w_result = space.newbytes(result[start:end])
+    return space.newtuple([w_result, space.newint(len(data))])
 
 @unwrap_spec(data=str, errors='str_or_None')
 def escape_decode(space, data, errors='strict'):
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
     result = PyString_DecodeEscape(space, data, errors, None)
-    return space.newtuple([space.newbytes(result), space.wrap(len(data))])
+    return space.newtuple([space.newbytes(result), space.newint(len(data))])
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -24,8 +24,8 @@
             raise wrap_unicodedecodeerror(space, e, input, self.name)
         except RuntimeError:
             raise wrap_runtimeerror(space)
-        return space.newtuple([space.wrap(output),
-                               space.wrap(len(input))])
+        return space.newtuple([space.newunicode(output),
+                               space.newint(len(input))])
 
     @unwrap_spec(input=unicode, errors="str_or_None")
     def encode(self, space, input, errors=None):
@@ -41,7 +41,7 @@
         except RuntimeError:
             raise wrap_runtimeerror(space)
         return space.newtuple([space.newbytes(output),
-                               space.wrap(len(input))])
+                               space.newint(len(input))])
 
 
 MultibyteCodec.typedef = TypeDef(
@@ -58,28 +58,28 @@
         codec = c_codecs.getcodec(name)
     except KeyError:
         raise oefmt(space.w_LookupError, "no such codec is supported.")
-    return space.wrap(MultibyteCodec(name, codec))
+    return MultibyteCodec(name, codec)
 
 
 def wrap_unicodedecodeerror(space, e, input, name):
     return OperationError(
         space.w_UnicodeDecodeError,
         space.newtuple([
-            space.wrap(name),
+            space.newtext(name),
             space.newbytes(input),
-            space.wrap(e.start),
-            space.wrap(e.end),
-            space.wrap(e.reason)]))
+            space.newint(e.start),
+            space.newint(e.end),
+            space.newtext(e.reason)]))
 
 def wrap_unicodeencodeerror(space, e, input, name):
     raise OperationError(
         space.w_UnicodeEncodeError,
         space.newtuple([
-            space.wrap(name),
-            space.wrap(input),
-            space.wrap(e.start),
-            space.wrap(e.end),
-            space.wrap(e.reason)]))
+            space.newtext(name),
+            space.newunicode(input),
+            space.newint(e.start),
+            space.newint(e.end),
+            space.newtext(e.reason)]))
 
 def wrap_runtimeerror(space):
     raise oefmt(space.w_RuntimeError, "internal codec error")

From pypy.commits at gmail.com  Wed Nov  2 09:03:51 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:51 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: weakref
Message-ID: <5819e437.958c1c0a.86993.ed08@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88076:149dea09020c
Date: 2016-11-02 13:26 +0100
http://bitbucket.org/pypy/pypy/changeset/149dea09020c/

Log:	weakref

diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -271,10 +271,10 @@
     """Return the number of weak references to 'obj'."""
     lifeline = w_obj.getweakref()
     if lifeline is None:
-        return space.wrap(0)
+        return space.newint(0)
     else:
         result = lifeline.traverse(_weakref_count, 0)
-        return space.wrap(result)
+        return space.newint(result)
 
 def _get_weakrefs(lifeline, wref, result):
     w_ref = wref()

From pypy.commits at gmail.com  Wed Nov  2 09:03:52 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:52 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: mmap
Message-ID: <5819e438.4f831c0a.30d34.de5d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88077:2f7fe8561763
Date: 2016-11-02 13:27 +0100
http://bitbucket.org/pypy/pypy/changeset/2f7fe8561763/

Log:	mmap

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -32,18 +32,18 @@
     def read_byte(self):
         self.check_valid()
         try:
-            return self.space.wrap(self.mmap.read_byte())
+            return self.space.newbytes(self.mmap.read_byte())
         except RValueError as v:
             raise mmap_error(self.space, v)
 
     def readline(self):
         self.check_valid()
-        return self.space.wrap(self.mmap.readline())
+        return self.space.newbytes(self.mmap.readline())
 
     @unwrap_spec(num=int)
     def read(self, num=-1):
         self.check_valid()
-        return self.space.wrap(self.mmap.read(num))
+        return self.space.newbytes(self.mmap.read(num))
 
     def find(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
@@ -57,7 +57,7 @@
             end = self.mmap.size
         else:
             end = space.getindex_w(w_end, None)
-        return space.wrap(self.mmap.find(tofind, start, end))
+        return space.newint(self.mmap.find(tofind, start, end))
 
     def rfind(self, w_tofind, w_start=None, w_end=None):
         self.check_valid()
@@ -71,7 +71,7 @@
             end = self.mmap.size
         else:
             end = space.getindex_w(w_end, None)
-        return space.wrap(self.mmap.find(tofind, start, end, True))
+        return space.newint(self.mmap.find(tofind, start, end, True))
 
     @unwrap_spec(pos=OFF_T, whence=int)
     def seek(self, pos, whence=0):
@@ -83,12 +83,12 @@
 
     def tell(self):
         self.check_valid()
-        return self.space.wrap(self.mmap.tell())
+        return self.space.newint(self.mmap.tell())
 
     def descr_size(self):
         self.check_valid()
         try:
-            return self.space.wrap(self.mmap.file_size())
+            return self.space.newint(self.mmap.file_size())
         except OSError as e:
             raise mmap_error(self.space, e)
 
@@ -114,7 +114,7 @@
     def flush(self, offset=0, size=0):
         self.check_valid()
         try:
-            return self.space.wrap(self.mmap.flush(offset, size))
+            return self.space.newint(self.mmap.flush(offset, size))
         except RValueError as v:
             raise mmap_error(self.space, v)
         except OSError as e:
@@ -141,10 +141,10 @@
             # obscure: in this case, RValueError translates to an app-level
             # SystemError.
             raise OperationError(self.space.w_SystemError,
-                                 self.space.wrap(e.message))
+                                 self.space.newtext(e.message))
 
     def __len__(self):
-        return self.space.wrap(self.mmap.size)
+        return self.space.newint(self.mmap.size)
 
     def check_valid(self):
         try:
@@ -170,16 +170,16 @@
         space = self.space
         start, stop, step, length = space.decode_index4(w_index, self.mmap.size)
         if step == 0:  # index only
-            return space.wrap(self.mmap.getitem(start))
+            return space.newbytes(self.mmap.getitem(start))
         elif step == 1:
             if stop - start < 0:
-                return space.wrap("")
-            return space.wrap(self.mmap.getslice(start, length))
+                return space.newbytes("")
+            return space.newbytes(self.mmap.getslice(start, length))
         else:
             b = StringBuilder(length)
             for i in range(start, stop, step):
                 b.append(self.mmap.getitem(i))
-            return space.wrap(b.build())
+            return space.newbytes(b.build())
 
     def descr_setitem(self, w_index, w_value):
         space = self.space
@@ -219,7 +219,7 @@
             j = i
         elif j > self.mmap.size:
             j = self.mmap.size
-        return space.wrap(self.mmap.getslice(i, (j - i)))
+        return space.newbytes(self.mmap.getslice(i, (j - i)))
 
     def descr_setslice(self, space, w_ilow, w_ihigh, w_item):
         self.check_valid()
@@ -261,7 +261,7 @@
             raise mmap_error(space, e)
         except RMMapError as e:
             raise mmap_error(space, e)
-        return space.wrap(self)
+        return self
 
 elif rmmap._MS_WINDOWS:
 
@@ -278,7 +278,7 @@
             raise mmap_error(space, e)
         except RMMapError as e:
             raise mmap_error(space, e)
-        return space.wrap(self)
+        return self
 
 W_MMap.typedef = TypeDef("mmap.mmap", None, None, "read-write",
     __new__ = interp2app(mmap),
@@ -319,15 +319,15 @@
 @objectmodel.dont_inline
 def mmap_error(space, e):
     if isinstance(e, RValueError):
-        return OperationError(space.w_ValueError, space.wrap(e.message))
+        return OperationError(space.w_ValueError, space.newtext(e.message))
     elif isinstance(e, RTypeError):
-        return OperationError(space.w_TypeError, space.wrap(e.message))
+        return OperationError(space.w_TypeError, space.newtext(e.message))
     elif isinstance(e, OSError):
         w_error = space.fromcache(Cache).w_error
         return wrap_oserror(space, e, w_exception_class=w_error)
     else:
         # bogus 'e'?
-        return OperationError(space.w_SystemError, space.wrap('%s' % e))
+        return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
 class MMapBuffer(Buffer):

From pypy.commits at gmail.com  Wed Nov  2 09:03:54 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:03:54 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _socket
Message-ID: <5819e43a.87101c0a.e4d70.6701@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88078:bf0813413c80
Date: 2016-11-02 14:02 +0100
http://bitbucket.org/pypy/pypy/changeset/bf0813413c80/

Log:	_socket

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -18,7 +18,7 @@
         res = rsocket.gethostname()
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(res)
+    return space.newtext(res)
 
 @unwrap_spec(host=str)
 def gethostbyname(space, host):
@@ -31,12 +31,12 @@
         ip = addr.get_host()
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 def common_wrapgethost(space, (name, aliases, address_list)):
-    aliases = [space.wrap(alias) for alias in aliases]
-    address_list = [space.wrap(addr.get_host()) for addr in address_list]
-    return space.newtuple([space.wrap(name),
+    aliases = [space.newtext(alias) for alias in aliases]
+    address_list = [space.newtext(addr.get_host()) for addr in address_list]
+    return space.newtuple([space.newtext(name),
                            space.newlist(aliases),
                            space.newlist(address_list)])
 
@@ -82,7 +82,7 @@
         port = rsocket.getservbyname(name, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(port)
+    return space.newint(port)
 
 @unwrap_spec(port=int, w_proto = WrappedDefault(None))
 def getservbyport(space, port, w_proto):
@@ -104,7 +104,7 @@
         service = rsocket.getservbyport(port, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(service)
+    return space.newtext(service)
 
 @unwrap_spec(name=str)
 def getprotobyname(space, name):
@@ -116,7 +116,7 @@
         proto = rsocket.getprotobyname(name)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(proto)
+    return space.newint(proto)
 
 @unwrap_spec(flags=int)
 def getnameinfo(space, w_sockaddr, flags):
@@ -128,7 +128,7 @@
         host, servport = rsocket.getnameinfo(addr, flags)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.newtuple([space.wrap(host), space.wrap(servport)])
+    return space.newtuple([space.newtext(host), space.newtext(servport)])
 
 @unwrap_spec(fd=int, family=int, type=int, proto=int)
 def fromfd(space, fd, family, type, proto=0):
@@ -141,7 +141,7 @@
         sock = rsocket.fromfd(fd, family, type, proto)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(W_Socket(space, sock))
+    return W_Socket(space, sock)
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -159,8 +159,8 @@
     except SocketError as e:
         raise converted_error(space, e)
     return space.newtuple([
-        space.wrap(W_Socket(space, sock1)),
-        space.wrap(W_Socket(space, sock2))
+        W_Socket(space, sock1),
+        W_Socket(space, sock2)
     ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
@@ -172,7 +172,7 @@
 
     Convert a 16-bit integer from network to host byte order.
     """
-    return space.wrap(rsocket.ntohs(intmask(x)))
+    return space.newint(rsocket.ntohs(intmask(x)))
 
 @unwrap_spec(x="c_uint")
 def ntohl(space, x):
@@ -180,7 +180,7 @@
 
     Convert a 32-bit integer from network to host byte order.
     """
-    return space.wrap(rsocket.ntohl(x))
+    return space.newint(rsocket.ntohl(x))
 
 @unwrap_spec(x="c_uint")
 def htons(space, x):
@@ -188,7 +188,7 @@
 
     Convert a 16-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htons(intmask(x)))
+    return space.newint(rsocket.htons(intmask(x)))
 
 @unwrap_spec(x="c_uint")
 def htonl(space, x):
@@ -196,7 +196,7 @@
 
     Convert a 32-bit integer from host to network byte order.
     """
-    return space.wrap(rsocket.htonl(x))
+    return space.newint(rsocket.htonl(x))
 
 @unwrap_spec(ip=str)
 def inet_aton(space, ip):
@@ -221,7 +221,7 @@
         ip = rsocket.inet_ntoa(packed)
     except SocketError as e:
         raise converted_error(space, e)
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 @unwrap_spec(family=int, ip=str)
 def inet_pton(space, family, ip):
@@ -249,7 +249,7 @@
     except ValueError:
         raise oefmt(space.w_ValueError,
                     "invalid length of packed IP address string")
-    return space.wrap(ip)
+    return space.newtext(ip)
 
 @unwrap_spec(family=int, socktype=int, proto=int, flags=int)
 def getaddrinfo(space, w_host, w_port,
@@ -265,7 +265,7 @@
     elif space.isinstance_w(w_host, space.w_str):
         host = space.bytes_w(w_host)
     elif space.isinstance_w(w_host, space.w_unicode):
-        w_shost = space.call_method(w_host, "encode", space.wrap("idna"))
+        w_shost = space.call_method(w_host, "encode", space.newtext("idna"))
         host = space.bytes_w(w_shost)
     else:
         raise oefmt(space.w_TypeError,
@@ -286,10 +286,10 @@
                                   proto, flags)
     except SocketError as e:
         raise converted_error(space, e)
-    lst1 = [space.newtuple([space.wrap(family),
-                            space.wrap(socktype),
-                            space.wrap(protocol),
-                            space.wrap(canonname),
+    lst1 = [space.newtuple([space.newint(family),
+                            space.newint(socktype),
+                            space.newint(protocol),
+                            space.newtext(canonname),
                             addr_as_object(addr, INVALID_SOCKET, space)]) # -1 as per cpython
             for (family, socktype, protocol, canonname, addr) in lst]
     return space.newlist(lst1)
@@ -304,7 +304,7 @@
     timeout = rsocket.getdefaulttimeout()
     if timeout < 0.0:
         return space.w_None
-    return space.wrap(timeout)
+    return space.newfloat(timeout)
 
 def setdefaulttimeout(space, w_timeout):
     if space.is_w(w_timeout, space.w_None):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -16,27 +16,27 @@
 )
 
 
-# XXX Hack to seperate rpython and pypy
+# XXX Hack to separate rpython and pypy
 def addr_as_object(addr, fd, space):
     if isinstance(addr, rsocket.INETAddress):
-        return space.newtuple([space.wrap(addr.get_host()),
-                               space.wrap(addr.get_port())])
+        return space.newtuple([space.newtext(addr.get_host()),
+                               space.newint(addr.get_port())])
     elif isinstance(addr, rsocket.INET6Address):
-        return space.newtuple([space.wrap(addr.get_host()),
-                               space.wrap(addr.get_port()),
-                               space.wrap(addr.get_flowinfo()),
-                               space.wrap(addr.get_scope_id())])
+        return space.newtuple([space.newtext(addr.get_host()),
+                               space.newint(addr.get_port()),
+                               space.wrap(addr.get_flowinfo()), # YYY
+                               space.wrap(addr.get_scope_id())]) # YYY
     elif rsocket.HAS_AF_PACKET and isinstance(addr, rsocket.PacketAddress):
-        return space.newtuple([space.wrap(addr.get_ifname(fd)),
-                               space.wrap(addr.get_protocol()),
-                               space.wrap(addr.get_pkttype()),
-                               space.wrap(addr.get_hatype()),
-                               space.wrap(addr.get_haddr())])
+        return space.newtuple([space.newtext(addr.get_ifname(fd)),
+                               space.newint(addr.get_protocol()),
+                               space.newint(addr.get_pkttype()),
+                               space.newint(addr.get_hatype()),
+                               space.newtext(addr.get_haddr())])
     elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress):
-        return space.wrap(addr.get_path())
+        return space.newtext(addr.get_path())
     elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress):
-        return space.newtuple([space.wrap(addr.get_pid()),
-                               space.wrap(addr.get_groups())])
+        return space.newtuple([space.newint(addr.get_pid()),
+                               space.newint(addr.get_groups())])
     # If we don't know the address family, don't raise an
     # exception -- return it as a tuple.
     from rpython.rlib import _rsocket_rffi as _c
@@ -45,8 +45,8 @@
     datalen = addr.addrlen - rsocket.offsetof(_c.sockaddr, 'c_sa_data')
     rawdata = ''.join([a.c_sa_data[i] for i in range(datalen)])
     addr.unlock()
-    return space.newtuple([space.wrap(family),
-                          space.wrap(rawdata)])
+    return space.newtuple([space.newint(family),
+                           space.newtext(rawdata)])
 
 # XXX Hack to seperate rpython and pypy
 # XXX a bit of code duplication
@@ -144,7 +144,7 @@
 
 # XXX Hack to seperate rpython and pypy
 def ipaddr_from_object(space, w_sockaddr):
-    host = space.str_w(space.getitem(w_sockaddr, space.wrap(0)))
+    host = space.str_w(space.getitem(w_sockaddr, space.newint(0)))
     addr = rsocket.makeipaddr(host)
     fill_from_object(addr, space, w_sockaddr)
     return addr
@@ -166,21 +166,21 @@
         if is_open and self.space.sys.track_resources:
             w_repr = self.space.repr(self)
             str_repr = self.space.str_w(w_repr)
-            w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
+            w_msg = self.space.newtext("WARNING: unclosed " + str_repr)
             self.space.resource_warning(w_msg, self.w_tb)
 
     def get_type_w(self, space):
-        return space.wrap(self.sock.type)
+        return space.newint(self.sock.type)
 
     def get_proto_w(self, space):
-        return space.wrap(self.sock.proto)
+        return space.newint(self.sock.proto)
 
     def get_family_w(self, space):
-        return space.wrap(self.sock.family)
+        return space.newint(self.sock.family)
 
     def descr_repr(self, space):
         fd = intmask(self.sock.fd)  # Force to signed type even on Windows.
-        return space.wrap("<socket object, fd=%d, family=%d,"
+        return space.newtext("<socket object, fd=%d, family=%d,"
                           " type=%d, protocol=%d>" %
                           (fd, self.sock.family,
                            self.sock.type, self.sock.proto))
@@ -196,7 +196,7 @@
             fd, addr = self.sock.accept()
             sock = rsocket.make_socket(
                 fd, self.sock.family, self.sock.type, self.sock.proto)
-            return space.newtuple([space.wrap(W_Socket(space, sock)),
+            return space.newtuple([W_Socket(space, sock),
                                    addr_as_object(addr, sock.fd, space)])
         except SocketError as e:
             raise converted_error(space, e)
@@ -256,7 +256,7 @@
         except SocketError as e:
             raise converted_error(space, e)
         error = self.sock.connect_ex(addr)
-        return space.wrap(error)
+        return space.newint(error)
 
     def dup_w(self, space):
         try:
@@ -270,7 +270,7 @@
 
         Return the integer file descriptor of the socket.
         """
-        return space.wrap(intmask(self.sock.fd))
+        return space.newint(intmask(self.sock.fd))
 
     def getpeername_w(self, space):
         """getpeername() -> address info
@@ -306,7 +306,7 @@
         """
         if w_buflen is None:
             try:
-                return space.wrap(self.sock.getsockopt_int(level, optname))
+                return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
         buflen = space.int_w(w_buflen)
@@ -321,7 +321,7 @@
         timeout = self.sock.gettimeout()
         if timeout < 0.0:
             return space.w_None
-        return space.wrap(timeout)
+        return space.newfloat(timeout)
 
     @unwrap_spec(backlog="c_int")
     def listen_w(self, space, backlog):
@@ -389,7 +389,7 @@
             count = self.sock.send(data, flags)
         except SocketError as e:
             raise converted_error(space, e)
-        return space.wrap(count)
+        return space.newint(count)
 
     @unwrap_spec(data='bufferstr', flags=int)
     def sendall_w(self, space, data, flags=0):
@@ -426,7 +426,7 @@
             count = self.sock.sendto(data, flags, addr)
         except SocketError as e:
             raise converted_error(space, e)
-        return space.wrap(count)
+        return space.newint(count)
 
     @unwrap_spec(flag=bool)
     def setblocking_w(self, flag):
@@ -484,7 +484,7 @@
         if nbytes == 0 or nbytes > lgt:
             nbytes = lgt
         try:
-            return space.wrap(self.sock.recvinto(rwbuffer, nbytes, flags))
+            return space.newint(self.sock.recvinto(rwbuffer, nbytes, flags))
         except SocketError as e:
             raise converted_error(space, e)
 
@@ -503,7 +503,7 @@
                 w_addr = addr_as_object(addr, self.sock.fd, space)
             else:
                 w_addr = space.w_None
-            return space.newtuple([space.wrap(readlgt), w_addr])
+            return space.newtuple([space.newint(readlgt), w_addr])
         except SocketError as e:
             raise converted_error(space, e)
 
@@ -544,7 +544,7 @@
                 if value_ptr:
                     lltype.free(value_ptr, flavor='raw')
 
-            return space.wrap(recv_ptr[0])
+            return space.wrap(recv_ptr[0]) # YYY
         finally:
             lltype.free(recv_ptr, flavor='raw')
 
@@ -605,7 +605,7 @@
     except SocketError as e:
         raise converted_error(space, e)
     W_Socket.__init__(self, space, sock)
-    return space.wrap(self)
+    return self
 descr_socket_new = interp2app(newsocket)
 
 
@@ -678,10 +678,10 @@
     message = e.get_msg()
     w_exception_class = get_error(space, e.applevelerrcls)
     if isinstance(e, SocketErrorWithErrno):
-        w_exception = space.call_function(w_exception_class, space.wrap(e.errno),
-                                      space.wrap(message))
+        w_exception = space.call_function(w_exception_class, space.newint(e.errno),
+                                      space.newtext(message))
     else:
-        w_exception = space.call_function(w_exception_class, space.wrap(message))
+        w_exception = space.call_function(w_exception_class, space.newtext(message))
     return OperationError(w_exception_class, w_exception)
 
 # ____________________________________________________________

From pypy.commits at gmail.com  Wed Nov  2 09:12:11 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:12:11 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: gc, imp
Message-ID: <5819e62b.4d881c0a.daaaf.f503@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88079:22c094827f52
Date: 2016-11-02 14:08 +0100
http://bitbucket.org/pypy/pypy/changeset/22c094827f52/

Log:	gc, imp

diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -30,7 +30,7 @@
 def wrap(space, gcref):
     w_obj = try_cast_gcref_to_w_root(gcref)
     if w_obj is None:
-        w_obj = space.wrap(W_GcRef(gcref))
+        w_obj = W_GcRef(gcref)
     return w_obj
 
 def unwrap(space, w_obj):
@@ -149,7 +149,7 @@
     size = rgc.get_rpy_memory_usage(gcref)
     if size < 0:
         raise missing_operation(space)
-    return space.wrap(size)
+    return space.newint(size)
 
 def get_rpy_type_index(space, w_obj):
     """Return an integer identifying the RPython type of the given
@@ -159,7 +159,7 @@
     index = rgc.get_rpy_type_index(gcref)
     if index < 0:
         raise missing_operation(space)
-    return space.wrap(index)
+    return space.newint(index)
 
 def get_objects(space):
     """Return a list of all app-level objects."""
@@ -202,9 +202,9 @@
 def get_typeids_z(space):
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
-    return space.wrap(s)
+    return space.newtext(s)
 
 def get_typeids_list(space):
     l = rgc.get_typeids_list()
-    list_w = [space.wrap(l[i]) for i in range(len(l))]
+    list_w = [space.newint(l[i]) for i in range(len(l))]
     return space.newlist(list_w)
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -9,15 +9,19 @@
 
 
 def get_suffixes(space):
-    w = space.wrap
     suffixes_w = []
     if importing.has_so_extension(space):
         suffixes_w.append(
-            space.newtuple([w(importing.get_so_extension(space)),
-                            w('rb'), w(importing.C_EXTENSION)]))
+            space.newtuple([space.newtext(importing.get_so_extension(space)),
+                            space.newtext('rb'),
+                            space.newtext(importing.C_EXTENSION)]))
     suffixes_w.extend([
-        space.newtuple([w('.py'), w('U'), w(importing.PY_SOURCE)]),
-        space.newtuple([w('.pyc'), w('rb'), w(importing.PY_COMPILED)]),
+        space.newtuple([space.newtext('.py'),
+                        space.newtext('U'),
+                        space.newtext(importing.PY_SOURCE)]),
+        space.newtuple([space.newtext('.pyc'),
+                        space.newtext('rb'),
+                        space.newtext(importing.PY_COMPILED)]),
         ])
     return space.newlist(suffixes_w)
 
@@ -30,7 +34,7 @@
     c = x & 0xff
     x >>= 8
     d = x & 0xff
-    return space.wrap(chr(a) + chr(b) + chr(c) + chr(d))
+    return space.newbytes(chr(a) + chr(b) + chr(c) + chr(d))
 
 def get_file(space, w_file, filename, filemode):
     if space.is_none(w_file):

From pypy.commits at gmail.com  Wed Nov  2 09:12:13 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 02 Nov 2016 06:12:13 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: merge default
Message-ID: <5819e62d.e5bfc20a.b3270.d88f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88080:ee5ad76cdf97
Date: 2016-11-02 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/ee5ad76cdf97/

Log:	merge default

diff too long, truncating to 2000 out of 3090 lines

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -29,7 +29,7 @@
 class deque(object):
 
     def __new__(cls, iterable=(), *args, **kw):
-        self = super(deque, cls).__new__(cls, *args, **kw)
+        self = super(deque, cls).__new__(cls)
         self.clear()
         return self
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -229,7 +229,7 @@
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
-        self = super(_CData, cls).__new__(cls, *args, **kwds)
+        self = super(_CData, cls).__new__(cls)
         if '_abstract_' in cls.__dict__:
             raise TypeError("abstract class")
         if hasattr(cls, '_ffistruct_'):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -334,6 +334,8 @@
                         realtype, quals = self._get_type_and_quals(
                             decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
                 else:
                     raise api.CDefError("unrecognized construct", decl)
         except api.FFIError as e:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,10 +519,18 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            raise api.CDefError("%r has no values explicitly defined: "
-                                "refusing to guess which integer type it is "
-                                "meant to be (unsigned/signed, int/long)"
-                % self._get_c_name())
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -1,4 +1,5 @@
 import os
+import sys
 
 try:
     basestring
@@ -74,8 +75,13 @@
     Add py_limited_api to kwds if setuptools >= 26 is in use.
     Do not alter the setting if it already exists.
     Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+
+    CPython itself should ignore the flag in a debugging version
+    (by not listing .abi3.so in the extensions it supports), but
+    it doesn't so far, creating troubles.  That's why we check
+    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds:
+    if 'py_limited_api' not in kwds and not sys.flags.debug:
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -397,3 +397,13 @@
 in auto-generated C code, and at least some knowledge about the
 various components involved, from PyPy's own RPython source code to
 the GC and possibly the JIT.
+
+
+Why doesn't PyPy move to GitHub, Gitlab, ...?
+----------------------------------------------
+
+We've been quite happy with bitbucket.org. Moving version control systems and
+hosting is a lot of hard work: On the one hand, PyPy's mercurial history is
+long and gnarly. On the other hand, all our infrastructure (buildbots,
+benchmarking, etc) would have to be adapted. So unless somebody steps up and
+volunteers to do all that work, it will likely not happen.
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -1,8 +1,17 @@
 Downloading and Installing PyPy
 ===============================
 
+Using a packaged PyPy
+~~~~~~~~~~~~~~~~~~~~~
+
+Some Linux distributions provide a pypy package. Note that in order to
+install additional modules that require compilation, you may need to install
+additional packages such as pypy-dev. This will manifest as an error about
+"missing Python.h". Distributions do not as of yet supply many pypy-ready
+packages, if you require additional modules we recommend creating a virtualenv
+and using pip. 
+
 .. _prebuilt-pypy:
-
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -38,6 +47,9 @@
 and not move the binary there, else PyPy would not be able to find its
 library.
 
+Installing more modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
 If you want to install 3rd party libraries, the most convenient way is
 to install pip_ using ensurepip_ (unless you want to install virtualenv as 
 explained below; then you can directly use pip inside virtualenvs):
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -0,0 +1,147 @@
+============
+PyPy2.7 v5.6
+============
+
+We have released PyPy2.7 v5.6, about two months after PyPy2.7 v5.4.
+This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
+
+We continue to make incremental improvements to our C-API
+compatability layer (cpyext). We pass all but a few of the tests in the
+upstream numpy `test suite`_. 
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, and some of those
+changes have been backported to PyPy2.7 where relevant
+
+We changed ``timeit`` to now report average +- standard deviation, which is
+better than the misleading minimum value reported in CPython.
+
+We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+well as maintaining support for previous versions.
+
+XXX
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.6 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _cffi: https://cffi.readthedocs.org
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://pypyjs.org
+
+Other Highlights (since 5.4 released Aug 31, 2016)
+=========================================================
+
+* New features
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+  * Backport fixes to rposix on windows from py3.5
+  * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
+  * Add ``inode`` to ``scandir()`` on posix systems
+  * Support more attributes on ``super``
+  * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
+  * Restore the ability to translate with CPython
+  * Update to CFFI 1.8.4
+  * Support the new buffer protocol in cpyext and numpypy
+  * Add ``rposix.sync()``
+  * Support full-precision nanosecond times in os.stat()
+  * Add documentation about the assembler backends to RPYthon
+  * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
+    changes downstream packaging requirments
+  * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
+  * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
+    be ``str`` or ``unicode``)
+  * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
+  * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
+    relevant platforms
+  * Backport the ``'faulthandler`` module from py3.5
+  * Improve the error message when trying to call a method where the ``self``
+    parameter is missing in the definition
+  * Implement ``rposix.cpu_count``
+  * Support translation on FreeBSD running on PowerPC
+  * Implement ``__rmod__`` on ``str`` and ``unicode`` types
+  * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
+
+* Bug Fixes
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Fix translation of the sandbox
+  * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+  * fix some calls to functions through window's COM interface
+  * fix minor leak when the C call to socketpair() fails
+  * make sure (-1.0 + 0j).__hash__(), (-1.0).__hash__() returns -2
+  * Fix for an issue where PyBytesResize was called on a fresh pyobj
+  * Fix bug in codewriter about passing the ``exitswitch`` variable to a call
+  * Don't crash in ``merge_if_blocks`` if the values are symbolics
+  * Issue #2325/2361: __class__ assignment between two classes with the same
+    slots
+  * Issue #2409: don't leak the file descriptor when doing ``open('some-dir')``
+  * Windows fixes around vmprof
+  * Don't use ``sprintf()`` from inside a signal handler
+  * Test and fix bug from the ``guard_not_forced_2`` branch, which didn't
+    save the floating-point register
+  * ``_numpypy.add.reduce`` returns a scalar now
+
+* Performance improvements:
+  * Improve method calls on oldstyle classes
+  * Clean and refactor code for testing cpyext to allow sharing with py3.5
+  * Refactor a building the map of reflected ops in ``_numpypy``
+  * Improve merging of virtual states in the JIT in order to avoid jumping to the
+    preamble
+  * In JIT residual calls, if the called function starts with a fast-path like
+    ``if x.foo != 0: return x.foo``, then inline the check before doing the
+     ``CALL``.
+  * Ensure ``make_inputargs`` fails properly when given arguments with type 
+    information
+  * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
+    about more easily and debugging is faster
+  * Refactor and remove dead code from ``optimizeopt``, ``resume``
+  
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.6.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -80,3 +80,20 @@
 
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: newinitwarn
+
+Issue warnings for stricter handling of ``__new/init__`` args (that
+become TypeErrors in python 3)
+
+.. branch: openssl-1.1
+
+PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
+installed.  Thanks tumbleweed!
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -72,10 +72,7 @@
             exc_typename = str(self.w_type)
             exc_value = str(w_value)
         else:
-            if space.is_w(space.type(self.w_type), space.w_str):
-                exc_typename = space.str_w(self.w_type)
-            else:
-                exc_typename = space.str_w(
+            exc_typename = space.str_w(
                     space.getattr(self.w_type, space.newtext('__name__')))
             if space.is_w(w_value, space.w_None):
                 exc_value = ""
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -610,7 +610,7 @@
 
 class StaticMethod(W_Root):
     """The staticmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -621,13 +621,16 @@
 
     def descr_staticmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(StaticMethod, w_subtype)
-        instance.__init__(w_function)
+        instance.__init__(space.w_None)
         return instance
 
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
+
 
 class ClassMethod(W_Root):
     """The classmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -640,9 +643,12 @@
 
     def descr_classmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(ClassMethod, w_subtype)
-        instance.__init__(w_function)
+        instance.__init__(space.w_None)
         return instance
 
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
+
 class FunctionWithFixedCode(Function):
     can_change_code = False
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
@@ -687,15 +689,17 @@
 (e.g. C().f()).  The instance is ignored except for its class.""",
     __get__ = interp2app(StaticMethod.descr_staticmethod_get),
     __new__ = interp2app(StaticMethod.descr_staticmethod__new__.im_func),
+    __init__=interp2app(StaticMethod.descr_init),
     __func__= interp_attrproperty_w('w_function', cls=StaticMethod),
     )
 
 ClassMethod.typedef = TypeDef(
     'classmethod',
-    __new__ = interp2app(ClassMethod.descr_classmethod__new__.im_func),
-    __get__ = interp2app(ClassMethod.descr_classmethod_get),
-    __func__= interp_attrproperty_w('w_function', cls=ClassMethod),
-    __doc__ = """classmethod(function) -> class method
+    __new__=interp2app(ClassMethod.descr_classmethod__new__.im_func),
+    __init__=interp2app(ClassMethod.descr_init),
+    __get__=interp2app(ClassMethod.descr_classmethod_get),
+    __func__=interp_attrproperty_w('w_function', cls=ClassMethod),
+    __doc__="""classmethod(function) -> class method
 
 Convert a function to be a class method.
 
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -20,6 +20,12 @@
         x = Static(1)
         assert isinstance(x, Static)
 
+        class C(Static):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_classmethod(self):
         class C(object):
             def f(cls, stuff):
@@ -41,8 +47,14 @@
         x = Classm(1)
         assert isinstance(x, Classm)
 
+        class C(Classm):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_property_simple(self):
-        
+
         class a(object):
             def _get(self): return 42
             def _set(self, value): raise AttributeError
@@ -98,7 +110,7 @@
             assert message.startswith('super(type, obj): obj must be an instance or subtype of type')
 
     def test_super_various(self):
-        
+
         class A(object):
             def meth(self, a):
                 return "A(%r)" % a
@@ -352,10 +364,10 @@
 
     def test_property_subclass_with_init(self):
         l = []
-        
+
         def x(self):
             l.append('x')
-        
+
         class P(property):
             def __init__(self):
                 property.__init__(self, x)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.4"
+VERSION = "1.9.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
+from pypy.module._cffi_backend import ctypestruct
 from pypy.objspace.std.bufferobject import W_Buffer
 
 from rpython.rlib.buffer import Buffer
@@ -71,7 +72,12 @@
     ctype = w_cdata.ctype
     if isinstance(ctype, ctypeptr.W_CTypePointer):
         if size < 0:
-            size = ctype.ctitem.size
+            structobj = w_cdata.get_structobj()
+            if (structobj is not None and
+                isinstance(structobj.ctype, ctypestruct.W_CTypeStructOrUnion)):
+                size = structobj._sizeof()
+            if size < 0:
+                size = ctype.ctitem.size
     elif isinstance(ctype, ctypearray.W_CTypeArray):
         if size < 0:
             size = w_cdata._sizeof()
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -329,7 +329,7 @@
     def getattr(self, w_attr):
         cfield = self.getcfield(w_attr)
         with self as ptr:
-            w_res = cfield.read(ptr)
+            w_res = cfield.read(ptr, self)
         return w_res
 
     def setattr(self, w_attr, w_value):
@@ -432,6 +432,9 @@
         lst = ct.cdata_dir()
         return space.newlist([space.wrap(s) for s in lst])
 
+    def get_structobj(self):
+        return None
+
 
 class W_CDataMem(W_CData):
     """This is used only by the results of cffi.cast('int', x)
@@ -453,28 +456,36 @@
     by newp().  They create and free their own memory according to an
     allocator."""
 
-    # the 'length' is either >= 0 for arrays, or -1 for pointers.
-    _attrs_ = ['length']
-    _immutable_fields_ = ['length']
+    # the 'allocated_length' is >= 0 for arrays; for var-sized
+    # structures it is the total size in bytes; otherwise it is -1.
+    _attrs_ = ['allocated_length']
+    _immutable_fields_ = ['allocated_length']
 
     def __init__(self, space, cdata, ctype, length=-1):
         W_CData.__init__(self, space, cdata, ctype)
-        self.length = length
+        self.allocated_length = length
 
     def _repr_extra(self):
         return self._repr_extra_owning()
 
     def _sizeof(self):
         ctype = self.ctype
-        if self.length >= 0:
+        if self.allocated_length >= 0:
             from pypy.module._cffi_backend import ctypearray
-            assert isinstance(ctype, ctypearray.W_CTypeArray)
-            return self.length * ctype.ctitem.size
+            if isinstance(ctype, ctypearray.W_CTypeArray):
+                return self.allocated_length * ctype.ctitem.size
+            else:
+                return self.allocated_length    # var-sized struct size
         else:
             return ctype.size
 
     def get_array_length(self):
-        return self.length
+        from pypy.module._cffi_backend import ctypearray
+        assert isinstance(self.ctype, ctypearray.W_CTypeArray)
+        return self.allocated_length
+
+    def get_structobj(self):
+        return self
 
 
 class W_CDataNewStd(W_CDataNewOwning):
@@ -508,12 +519,19 @@
         self.structobj = structobj
 
     def _repr_extra(self):
-        return self._repr_extra_owning()
+        return self.structobj._repr_extra_owning()
 
     def _do_getitem(self, ctype, i):
         assert i == 0
         return self.structobj
 
+    def get_structobj(self):
+        structobj = self.structobj
+        if isinstance(structobj, W_CDataNewOwning):
+            return structobj
+        else:
+            return None
+
 
 class W_CDataSliced(W_CData):
     """Subclass with an explicit length, for slices."""
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -211,13 +211,16 @@
             # a W_CDataPtrToStruct object which has a strong reference
             # to a W_CDataNewOwning that really contains the structure.
             #
-            if not space.is_w(w_init, space.w_None):
-                ctitem.force_lazy_struct()
-                if ctitem._with_var_array:
+            varsize_length = -1
+            ctitem.force_lazy_struct()
+            if ctitem._with_var_array:
+                if not space.is_w(w_init, space.w_None):
                     datasize = ctitem.convert_struct_from_object(
                         lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
+                varsize_length = datasize
             #
-            cdatastruct = allocator.allocate(space, datasize, ctitem)
+            cdatastruct = allocator.allocate(space, datasize, ctitem,
+                                             length=varsize_length)
             ptr = cdatastruct.unsafe_escaping_ptr()
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
                                                        self, cdatastruct)
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -210,7 +210,7 @@
         return W_CField(self.ctype, offset + self.offset,
                         self.bitshift, self.bitsize, self.flags | fflags)
 
-    def read(self, cdata):
+    def read(self, cdata, w_cdata):
         cdata = rffi.ptradd(cdata, self.offset)
         if self.bitshift == self.BS_REGULAR:
             return self.ctype.convert_to_object(cdata)
@@ -218,6 +218,14 @@
             from pypy.module._cffi_backend import ctypearray
             ctype = self.ctype
             assert isinstance(ctype, ctypearray.W_CTypeArray)
+            structobj = w_cdata.get_structobj()
+            if structobj is not None:
+                # variable-length array
+                size = structobj.allocated_length - self.offset
+                if size >= 0:
+                    arraylen = size // ctype.ctitem.size
+                    return cdataobj.W_CDataSliced(ctype.space, cdata, ctype,
+                                                  arraylen)
             return cdataobj.W_CData(ctype.space, cdata, ctype.ctptr)
         else:
             return self.convert_bitfield_to_object(cdata)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -353,7 +353,7 @@
         if fbitsize < 0:
             # not a bitfield: common case
 
-            if isinstance(ftype, ctypearray.W_CTypeArray) and ftype.length==0:
+            if isinstance(ftype, ctypearray.W_CTypeArray) and ftype.length<=0:
                 bs_flag = ctypestruct.W_CField.BS_EMPTY_ARRAY
             else:
                 bs_flag = ctypestruct.W_CField.BS_REGULAR
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3161,17 +3161,19 @@
     assert d[1][0] == 'y'
     assert d[1][1].type is BArray
     assert d[1][1].offset == size_of_int()
-    assert d[1][1].bitshift == -1
+    assert d[1][1].bitshift == -2
     assert d[1][1].bitsize == -1
     #
     p = newp(new_pointer_type(BStruct))
     p.x = 42
     assert p.x == 42
-    assert typeof(p.y) is BIntP
+    assert typeof(p.y) is BArray
+    assert len(p.y) == 0
     assert p.y == cast(BIntP, p) + 1
     #
     p = newp(new_pointer_type(BStruct), [100])
     assert p.x == 100
+    assert len(p.y) == 0
     #
     # Tests for
     #    ffi.new("struct_with_var_array *", [field.., [the_array_items..]])
@@ -3186,6 +3188,10 @@
             p.y[0] = 200
             assert p.y[2] == 0
             p.y[2] = 400
+        assert len(p.y) == 3
+        assert len(p[0].y) == 3
+        assert len(buffer(p)) == sizeof(BInt) * 4
+        assert sizeof(p[0]) == sizeof(BInt) * 4
         plist.append(p)
     for i in range(20):
         p = plist[i]
@@ -3193,13 +3199,31 @@
         assert p.y[0] == 200
         assert p.y[1] == i
         assert p.y[2] == 400
-        assert list(p.y[0:3]) == [200, i, 400]
+        assert list(p.y) == [200, i, 400]
     #
     # the following assignment works, as it normally would, for any array field
-    p.y = [500, 600]
-    assert list(p.y[0:3]) == [500, 600, 400]
+    p.y = [501, 601]
+    assert list(p.y) == [501, 601, 400]
+    p[0].y = [500, 600]
+    assert list(p[0].y) == [500, 600, 400]
+    assert repr(p) == "<cdata 'foo *' owning %d bytes>" % (
+        sizeof(BStruct) + 3 * sizeof(BInt),)
+    assert repr(p[0]) == "<cdata 'foo' owning %d bytes>" % (
+        sizeof(BStruct) + 3 * sizeof(BInt),)
+    assert sizeof(p[0]) == sizeof(BStruct) + 3 * sizeof(BInt)
+    #
+    # from a non-owning pointer, we can't get the length
+    q = cast(new_pointer_type(BStruct), p)
+    assert q.y[0] == 500
+    assert q[0].y[0] == 500
+    py.test.raises(TypeError, len, q.y)
+    py.test.raises(TypeError, len, q[0].y)
+    assert typeof(q.y) is BIntP
+    assert typeof(q[0].y) is BIntP
+    assert sizeof(q[0]) == sizeof(BStruct)
     #
     # error cases
+    py.test.raises(IndexError, "p.y[4]")
     py.test.raises(TypeError, "p.y = cast(BIntP, 0)")
     py.test.raises(TypeError, "p.y = 15")
     py.test.raises(TypeError, "p.y = None")
@@ -3264,6 +3288,33 @@
         assert p.x[5] == 60
         assert p.x[6] == 70
 
+def test_struct_array_not_aligned():
+    # struct a { int x; char y; char z[]; };
+    # ends up of size 8, but 'z' is at offset 5
+    BChar = new_primitive_type("char")
+    BInt = new_primitive_type("int")
+    BCharP = new_pointer_type(BChar)
+    BArray = new_array_type(BCharP, None)
+    BStruct = new_struct_type("foo")
+    complete_struct_or_union(BStruct, [('x', BInt),
+                                       ('y', BChar),
+                                       ('z', BArray)])
+    assert sizeof(BStruct) == 2 * size_of_int()
+    def offsetof(BType, fieldname):
+        return typeoffsetof(BType, fieldname)[1]
+    base = offsetof(BStruct, 'z')
+    assert base == size_of_int() + 1
+    #
+    p = newp(new_pointer_type(BStruct), {'z': 3})
+    assert sizeof(p[0]) == base + 3
+    q = newp(new_pointer_type(BStruct), {'z': size_of_int()})
+    assert sizeof(q) == size_of_ptr()
+    assert sizeof(q[0]) == base + size_of_int()
+    assert len(p.z) == 3
+    assert len(p[0].z) == 3
+    assert len(q.z) == size_of_int()
+    assert len(q[0].z) == size_of_int()
+
 def test_ass_slice():
     BChar = new_primitive_type("char")
     BArray = new_array_type(new_pointer_type(BChar), None)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -408,11 +408,14 @@
             'test_misdeclared_field_1',
             "struct foo_s { int a[6]; };")
         assert ffi.sizeof("struct foo_s") == 24  # found by the actual C code
-        p = ffi.new("struct foo_s *")
-        # lazily build the fields and boom:
-        e = raises(ffi.error, getattr, p, "a")
-        assert str(e.value).startswith("struct foo_s: wrong size for field 'a' "
-                                       "(cdef says 20, but C compiler says 24)")
+        try:
+            # lazily build the fields and boom:
+            p = ffi.new("struct foo_s *")
+            p.a
+            assert False, "should have raised"
+        except ffi.error as e:
+            assert str(e).startswith("struct foo_s: wrong size for field 'a' "
+                                     "(cdef says 20, but C compiler says 24)")
 
     def test_open_array_in_struct(self):
         ffi, lib = self.prepare(
@@ -420,8 +423,10 @@
             'test_open_array_in_struct',
             "struct foo_s { int b; int a[]; };")
         assert ffi.sizeof("struct foo_s") == 4
-        p = ffi.new("struct foo_s *", [5, [10, 20, 30]])
+        p = ffi.new("struct foo_s *", [5, [10, 20, 30, 40]])
         assert p.a[2] == 30
+        assert ffi.sizeof(p) == ffi.sizeof("void *")
+        assert ffi.sizeof(p[0]) == 5 * ffi.sizeof("int")
 
     def test_math_sin_type(self):
         ffi, lib = self.prepare(
@@ -954,6 +959,7 @@
             "struct foo_s { int x; int a[5][8]; int y; };")
         assert ffi.sizeof('struct foo_s') == 42 * ffi.sizeof('int')
         s = ffi.new("struct foo_s *")
+        assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
         assert ffi.sizeof(s.a) == 40 * ffi.sizeof('int')
         assert s.a[4][7] == 0
         raises(IndexError, 's.a[4][8]')
@@ -961,6 +967,18 @@
         assert ffi.typeof(s.a) == ffi.typeof("int[5][8]")
         assert ffi.typeof(s.a[0]) == ffi.typeof("int[8]")
 
+    def test_struct_array_guess_length_3(self):
+        ffi, lib = self.prepare(
+            "struct foo_s { int a[][...]; };",
+            'test_struct_array_guess_length_3',
+            "struct foo_s { int x; int a[5][7]; int y; };")
+        assert ffi.sizeof('struct foo_s') == 37 * ffi.sizeof('int')
+        s = ffi.new("struct foo_s *")
+        assert ffi.typeof(s.a) == ffi.typeof("int[][7]")
+        assert s.a[4][6] == 0
+        raises(IndexError, 's.a[4][7]')
+        assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]")
+
     def test_global_var_array_2(self):
         ffi, lib = self.prepare(
             "int a[...][...];",
diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names
+from pypy.module._hashlib.interp_hashlib import (
+    algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC)
 
 
 class Module(MixedModule):
@@ -13,6 +14,9 @@
     for name in algorithms:
         interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
 
+    if HAS_FAST_PKCS5_PBKDF2_HMAC:
+        interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac'
+
     def startup(self, space):
         w_meth_names = fetch_names(space)
         space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -8,7 +8,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.thread.os_lock import Lock
 
@@ -58,23 +58,26 @@
     def __init__(self, space, name, copy_from=NULL_CTX):
         self.name = name
         digest_type = self.digest_type_by_name(space)
-        self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
+        self.digest_size = ropenssl.EVP_MD_size(digest_type)
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
-        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
         rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
         try:
             if copy_from:
-                ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
+                    raise ValueError
             else:
                 ropenssl.EVP_DigestInit(ctx, digest_type)
             self.ctx = ctx
         except:
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
             raise
         self.register_finalizer(space)
 
@@ -82,8 +85,7 @@
         ctx = self.ctx
         if ctx:
             self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-            ropenssl.EVP_MD_CTX_cleanup(ctx)
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
 
     def digest_type_by_name(self, space):
         digest_type = ropenssl.EVP_get_digestbyname(self.name)
@@ -128,21 +130,26 @@
 
     def get_block_size(self, space):
         digest_type = self.digest_type_by_name(space)
-        block_size = rffi.getintfield(digest_type, 'c_block_size')
+        block_size = ropenssl.EVP_MD_block_size(digest_type)
         return space.wrap(block_size)
 
     def get_name(self, space):
         return space.wrap(self.name)
 
     def _digest(self, space):
-        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
+        try:
             with self.lock:
-                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx):
+                    raise ValueError
             digest_size = self.digest_size
             with rffi.scoped_alloc_buffer(digest_size) as buf:
                 ropenssl.EVP_DigestFinal(ctx, buf.raw, None)
-                ropenssl.EVP_MD_CTX_cleanup(ctx)
                 return buf.str(digest_size)
+        finally:
+            ropenssl.EVP_MD_CTX_free(ctx)
 
 
 W_Hash.typedef = TypeDef(
@@ -177,3 +184,27 @@
 for _name in algorithms:
     _newname = 'new_%s' % (_name,)
     globals()[_newname] = make_new_hash(_name, _newname)
+
+
+HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
+if HAS_FAST_PKCS5_PBKDF2_HMAC:
+    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+                 w_dklen=WrappedDefault(None))
+    def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
+        digest = ropenssl.EVP_get_digestbyname(name)
+        if not digest:
+            raise oefmt(space.w_ValueError, "unknown hash function")
+        if space.is_w(w_dklen, space.w_None):
+            dklen = ropenssl.EVP_MD_size(digest)
+        else:
+            dklen = space.int_w(w_dklen)
+        if dklen < 1:
+            raise oefmt(space.w_ValueError,
+                        "key length must be greater than 0.")
+        with rffi.scoped_alloc_buffer(dklen) as buf:
+            r = ropenssl.PKCS5_PBKDF2_HMAC(
+                password, len(password), salt, len(salt), rounds, digest,
+                dklen, buf.raw)
+            if not r:
+                raise ValueError
+            return space.wrap(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -15,17 +15,19 @@
 
     def test_attributes(self):
         import hashlib
-        for name, expected_size in {'md5': 16,
-                                    'sha1': 20,
-                                    'sha224': 28,
-                                    'sha256': 32,
-                                    'sha384': 48,
-                                    'sha512': 64,
-                                    }.items():
+        for name, (expected_size, expected_block_size) in {
+                'md5': (16, 64),
+                'sha1': (20, 64),
+                'sha224': (28, 64),
+                'sha256': (32, 64),
+                'sha384': (48, 128),
+                'sha512': (64, 128),
+                }.items():
             h = hashlib.new(name)
             assert h.name == name
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -46,6 +48,7 @@
             h = py_new(name)('')
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update('abc')
             h2 = h.copy()
@@ -108,3 +111,13 @@
             got.decode('hex')
             if expected is not None:
                 assert got == expected
+
+    def test_pbkdf2(self):
+        try:
+            from _hashlib import pbkdf2_hmac
+        except ImportError:
+            skip("Requires OpenSSL >= 1.1")
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -479,6 +479,14 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recv_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recv_into(buffer, [nbytes[, flags]]) -> nbytes_read
+
+        A version of recv() that stores its data into a buffer rather than creating
+        a new string.  Receive up to buffersize bytes from the socket.  If buffersize
+        is not specified (or 0), receive up to the size available in the given buffer.
+
+        See recv() for documentation about the flags.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0 or nbytes > lgt:
@@ -490,6 +498,10 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recvfrom_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)
+
+        Like recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0:
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -33,9 +33,10 @@
 PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
 
 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
+ PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
  PY_SSL_VERSION_TLS1_2) = range(6)
 
+
 SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
 SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
@@ -68,11 +69,12 @@
 constants["HAS_NPN"] = HAS_NPN
 constants["HAS_ALPN"] = HAS_ALPN
 
+constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
+constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 if not OPENSSL_NO_SSL3:
     constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 if HAVE_TLSv1_2:
     constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
@@ -111,7 +113,10 @@
         err_reason = libssl_ERR_GET_REASON(errcode)
         reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
         lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        msg = rffi.charp2str(libssl_ERR_reason_error_string(errcode))
+        raw_msg = libssl_ERR_reason_error_string(errcode)
+        msg = None
+        if raw_msg:
+            msg = rffi.charp2str(raw_msg)
     if not msg:
         msg = "unknown error"
     if reason_str and lib_str:
@@ -637,9 +642,12 @@
         if not self.ssl:
             return space.w_None
         comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method or intmask(comp_method[0].c_type) == NID_undef:
+        if not comp_method:
             return space.w_None
-        short_name = libssl_OBJ_nid2sn(comp_method[0].c_type)
+        method_type = intmask(libssl_COMP_get_type(comp_method))
+        if method_type == NID_undef:
+            return space.w_None
+        short_name = libssl_COMP_get_name(comp_method)
         if not short_name:
             return space.w_None
         return space.wrap(rffi.charp2str(short_name))
@@ -795,7 +803,7 @@
     for index in range(entry_count):
         entry = libssl_X509_NAME_get_entry(xname, index)
         # check to see if we've gotten to a new RDN
-        entry_level = intmask(entry[0].c_set)
+        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
         if rdn_level >= 0:
             if rdn_level != entry_level:
                 # yes, new RDN
@@ -846,8 +854,9 @@
                                 "No method for internalizing subjectAltName!'")
 
             with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                p_ptr[0] = ext[0].c_value.c_data
-                length = intmask(ext[0].c_value.c_length)
+                ext_value = libssl_X509_EXTENSION_get_data(ext)
+                p_ptr[0] = ext_value.c_data
+                length = intmask(ext_value.c_length)
                 null = lltype.nullptr(rffi.VOIDP.TO)
                 if method[0].c_it:
                     names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
@@ -967,10 +976,8 @@
     if OPENSSL_VERSION_NUMBER >= 0x10001000:
         # Calls x509v3_cache_extensions and sets up crldp
         libssl_X509_check_ca(certificate)
-        dps = certificate[0].c_crldp
-    else:
-        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-            certificate, NID_crl_distribution_points, None, None))
+    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+        certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
@@ -1268,14 +1275,14 @@
     @staticmethod
     @unwrap_spec(protocol=int)
     def descr_new(space, w_subtype, protocol):
-        if protocol == PY_SSL_VERSION_TLS1:
+        if protocol == PY_SSL_VERSION_TLS:
+            method = libssl_TLS_method()
+        elif protocol == PY_SSL_VERSION_TLS1:
             method = libssl_TLSv1_method()
         elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
             method = libssl_SSLv3_method()
         elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
             method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_SSL23:
-            method = libssl_SSLv23_method()
         elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
             method = libssl_TLSv1_1_method()
         elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
@@ -1303,6 +1310,7 @@
             # OpenSSL 1.0.2+), or use prime256v1 by default.
             # This is Apache mod_ssl's initialization
             # policy, so we should be safe.
+            # OpenSSL 1.1 has it enabled by default.
             if libssl_SSL_CTX_set_ecdh_auto:
                 libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
@@ -1390,20 +1398,22 @@
 
     def descr_get_verify_flags(self, space):
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         return space.wrap(flags)
 
     def descr_set_verify_flags(self, space, w_obj):
         new_flags = space.int_w(w_obj)
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         flags_clear = flags & ~new_flags
         flags_set = ~flags & new_flags
         if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                store[0].c_param, flags_clear):
+                param, flags_clear):
             raise _ssl_seterror(space, None, 0)
         if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                store[0].c_param, flags_set):
+                param, flags_set):
             raise _ssl_seterror(space, None, 0)
 
     def descr_get_check_hostname(self, space):
@@ -1614,14 +1624,16 @@
         x509 = 0
         x509_ca = 0
         crl = 0
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) == X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
+            if obj_type == X509_LU_X509:
                 x509 += 1
                 if libssl_X509_check_ca(
-                        libssl_pypy_X509_OBJECT_data_x509(obj)):
+                        libssl_X509_OBJECT_get0_X509(obj)):
                     x509_ca += 1
-            elif intmask(obj.c_type) == X509_LU_CRL:
+            elif obj_type == X509_LU_CRL:
                 crl += 1
             else:
                 # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
@@ -1660,13 +1672,14 @@
             binary_mode = False
         rlist = []
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) != X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
                 # not a x509 cert
                 continue
             # CA for any purpose
-            cert = libssl_pypy_X509_OBJECT_data_x509(obj)
+            cert = libssl_X509_OBJECT_get0_X509(obj)
             if not libssl_X509_check_ca(cert):
                 continue
             if binary_mode:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -77,7 +77,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+            ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
             ss = ctx._wrap_socket(s, 0)
             assert ss.context is ctx
         exc = raises(_socket.error, ss.do_handshake)
@@ -95,7 +95,7 @@
         if sys.version_info < (2, 7, 9):
             ss = _ssl.sslwrap(s, 0)
         else:
-            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0)
+            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLS)._wrap_socket(s, 0)
         s.close()
         exc = raises(_ssl.SSLError, ss.write, "data")
         assert exc.value.message == 'Underlying socket has been closed.'
@@ -123,13 +123,13 @@
 
     def test_context(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         raises(ValueError, _ssl._SSLContext, -1)
 
         assert type(s.options) is long
-        assert s.options & _ssl.OP_NO_SSLv2
-        s.options &= ~_ssl.OP_NO_SSLv2
-        assert not s.options & _ssl.OP_NO_SSLv2
+        assert s.options & _ssl.OP_NO_SSLv3
+        s.options &= ~_ssl.OP_NO_SSLv3
+        assert not s.options & _ssl.OP_NO_SSLv3
         raises(TypeError, "s.options = 2.5")
 
         assert not s.check_hostname
@@ -159,7 +159,7 @@
 
     def test_set_default_verify_paths(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s.set_default_verify_paths()
 
 
@@ -253,13 +253,44 @@
         if not _ssl.HAS_NPN:
             skip("NPN requires OpenSSL 1.0.1 or greater")
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s._sock, True,
                               server_hostname="svn.python.org")
         self.s.close()
         del ss; gc.collect()
 
+    def test_peer_certificate(self):
+        import _ssl, gc
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ss = ctx._wrap_socket(self.s._sock, False)
+        ss.do_handshake()
+        assert isinstance(ss.peer_certificate(der=True), bytes)
+        assert isinstance(ss.peer_certificate(), dict)
+        self.s.close()
+        del ss; gc.collect()
+
+    def test_peer_certificate_verify(self):
+        import _ssl, ssl, gc
+        paths = ssl.get_default_verify_paths()
+        if not paths.capath and not paths.cafile:
+            skip("ssl.get_default_verify_paths() failed to return any path")
+
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.verify_mode = _ssl.CERT_REQUIRED
+        ctx.load_verify_locations(capath=paths.capath, cafile=paths.cafile)
+
+        ss = ctx._wrap_socket(self.s._sock, False)
+        try:
+            ss.do_handshake()
+        except _ssl.SSLError as e:
+            if e.reason == 'CERTIFICATE_VERIFY_FAILED':
+                skip("Certificate verification failed. "
+                     "Most likely we just don't have any CA certificates.")
+        assert ss.peer_certificate()
+        self.s.close()
+        del ss; gc.collect()
+
     def test_tls_unique_cb(self):
         import ssl, sys, gc
         ss = ssl.wrap_socket(self.s)
@@ -325,7 +356,7 @@
 
     def test_load_cert_chain(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
         exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
@@ -344,11 +375,11 @@
 
     def test_load_verify_locations(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         ctx.load_verify_locations(cafile=self.keycert, capath=None)
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         with open(self.keycert) as f:
             cacert_pem = f.read().decode('ascii')
         ctx.load_verify_locations(cadata=cacert_pem)
@@ -356,7 +387,7 @@
 
     def test_get_ca_certs(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         assert ctx.get_ca_certs() == []
         ctx.load_verify_locations(self.python_org_cert)
@@ -370,7 +401,7 @@
 
     def test_cert_store_stats(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
         ctx.load_cert_chain(self.keycert)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
@@ -379,7 +410,7 @@
 
     def test_load_dh_params(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
@@ -389,7 +420,7 @@
 
     def test_set_ecdh_curve(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.set_ecdh_curve("prime256v1")
         raises(ValueError, ctx.set_ecdh_curve, "foo")
 
@@ -434,7 +465,7 @@
     def test_lib_reason(self):
         # Test the library and reason attributes
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         exc = raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
         assert exc.value.library == 'PEM'
         assert exc.value.reason == 'NO_START_LINE'
@@ -445,7 +476,7 @@
         # Check that the appropriate SSLError subclass is raised
         # (this only tests one of them)
         import _ssl, _socket
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s = _socket.socket()
         try:
             s.bind(("127.0.0.1", 0))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -2,7 +2,18 @@
 import pytest
 
 
-class BaseArrayTests:
+class AppTestArray(object):
+    spaceconfig = {'usemodules': ['array', 'struct', '_rawffi', 'binascii']}
+
+    def setup_class(cls):
+        cls.w_array = cls.space.appexec([], """():
+            import array
+            return array.array
+        """)
+        cls.w_tempfile = cls.space.wrap(
+            str(pytest.ensuretemp('array').join('tmpfile')))
+        cls.w_maxint = cls.space.wrap(sys.maxint)
+
     def test_ctor(self):
         assert len(self.array('c')) == 0
         assert len(self.array('i')) == 0
@@ -22,8 +33,8 @@
         a = self.array('u')
         raises(TypeError, a.append, 7)
         raises(TypeError, a.append, u'hi')
-        a.append(unicode('h'))
-        assert a[0] == unicode('h')
+        a.append(u'h')
+        assert a[0] == u'h'
         assert type(a[0]) is unicode
         assert len(a) == 1
 
@@ -67,7 +78,7 @@
                            ('H', (     0, 56783, 65535),  int),
                            ('i', (-32768, 30535, 32767),  int),
                            ('I', (     0, 56783, 65535), long),
-                           ('l', (-2 ** 32 / 2, 34, 2 ** 32 / 2 - 1),  int),
+                           ('l', (-2 ** 32 // 2, 34, 2 ** 32 // 2 - 1),  int),
                            ('L', (0, 3523532, 2 ** 32 - 1), long),
                            ):
             a = self.array(tc, ok)
@@ -105,7 +116,7 @@
             assert a.tolist() == vals
 
             a = self.array(tc.lower())
-            vals = [-1 * (2 ** itembits) / 2,  (2 ** itembits) / 2 - 1]
+            vals = [-1 * (2 ** itembits) // 2,  (2 ** itembits) // 2 - 1]
             a.fromlist(vals)
             assert a.tolist() == vals
 
@@ -137,11 +148,11 @@
         for t in inttypes:
             a = self.array(t, [1, 2, 3])
             b = a.itemsize
-            for v in (-2 ** (8 * b) / 2, 2 ** (8 * b) / 2 - 1):
+            for v in (-2 ** (8 * b) // 2, 2 ** (8 * b) // 2 - 1):
                 a[1] = v
                 assert a[0] == 1 and a[1] == v and a[2] == 3
-            raises(OverflowError, a.append, -2 ** (8 * b) / 2 - 1)
-            raises(OverflowError, a.append, 2 ** (8 * b) / 2)
+            raises(OverflowError, a.append, -2 ** (8 * b) // 2 - 1)
+            raises(OverflowError, a.append, 2 ** (8 * b) // 2)
 
             a = self.array(t.upper(), [1, 2, 3])
             b = a.itemsize
@@ -175,42 +186,35 @@
                 raises(ValueError, a.fromstring, '\x00' * (a.itemsize + 1))
                 raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize - 1))
                 raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize + 1))
-            b = self.array(t, '\x00' * a.itemsize * 2)
+            b = self.array(t, b'\x00' * a.itemsize * 2)
             assert len(b) == 2 and b[0] == 0 and b[1] == 0
             if sys.version_info >= (2, 7, 11):
                 raises(ValueError, a.fromstring, a)
 
     def test_fromfile(self):
-
-        ## class myfile(object):
-        ##     def __init__(self, c, s):
-        ##         self.c = c
-        ##         self.s = s
-        ##     def read(self,n):
-        ##         return self.c*min(n,self.s)
         def myfile(c, s):
-            f = open(self.tempfile, 'w')
+            f = open(self.tempfile, 'wb')
             f.write(c * s)
             f.close()
-            return open(self.tempfile, 'r')
+            return open(self.tempfile, 'rb')
 
-        f = myfile('\x00', 100)
+        f = myfile(b'\x00', 100)
         for t in 'bBhHiIlLfd':
             a = self.array(t)
             a.fromfile(f, 2)
             assert len(a) == 2 and a[0] == 0 and a[1] == 0
 
         a = self.array('b')
-        a.fromfile(myfile('\x01', 20), 2)
+        a.fromfile(myfile(b'\x01', 20), 2)
         assert len(a) == 2 and a[0] == 1 and a[1] == 1
 
         a = self.array('h')
-        a.fromfile(myfile('\x01', 20), 2)
+        a.fromfile(myfile(b'\x01', 20), 2)
         assert len(a) == 2 and a[0] == 257 and a[1] == 257
 
         for i in (0, 1):
             a = self.array('h')
-            raises(EOFError, a.fromfile, myfile('\x01', 2 + i), 2)
+            raises(EOFError, a.fromfile, myfile(b'\x01', 2 + i), 2)
             assert len(a) == 1 and a[0] == 257
 
     def test_fromlist(self):
@@ -250,12 +254,12 @@
         assert repr(a) == "array('b', [1, 2, 1, 2])"
 
     def test_fromunicode(self):
-        raises(ValueError, self.array('i').fromunicode, unicode('hi'))
+        raises(ValueError, self.array('i').fromunicode, u'hi')
         a = self.array('u')
-        a.fromunicode(unicode('hi'))
+        a.fromunicode(u'hi')
         assert len(a) == 2 and a[0] == 'h' and a[1] == 'i'
 
-        b = self.array('u', unicode('hi'))
+        b = self.array('u', u'hi')
         assert len(b) == 2 and b[0] == 'h' and b[1] == 'i'
 
     def test_sequence(self):
@@ -357,23 +361,6 @@
                         except ValueError:
                             assert not ok
 
-    def test_reversingslice_pre26(self):
-        import sys
-        if sys.version_info >= (2, 6):
-            skip('arrays can handle more slice ops than lists in 2.6')
-
-        for a in range(-4, 5):
-            for b in range(-4, 5):
-                for c in [-4, -3, -2, -1, 1, 2, 3, 4]:
-                    lst = [1, 2, 3]
-                    arr = self.array('i', lst)
-                    for vals in ([4, 5], [6], []):
-                        try:
-                            lst[a:b:c] = vals
-                        except ValueError:
-                            raises(ValueError,
-                                   "arr[a:b:c]=self.array('i', vals)")
-
     def test_toxxx(self):
         a = self.array('i', [1, 2, 3])
         l = a.tolist()
@@ -405,7 +392,7 @@
                                ('BHILfd', (127, 0, 1, 7, 255, 169)),
                                ('hilHILfd', (32760, 30123, 3422, 23244))):
             for tc in tcodes:
-                values += ((2 ** self.array(tc).itemsize) / 2 - 1, )
+                values += ((2 ** self.array(tc).itemsize) // 2 - 1, )
                 s = self.array(tc, values).tostring()
                 a = unpack(tc * len(values), s)
                 assert a == values
@@ -420,8 +407,7 @@
         assert repr(a) == "array('c', 'hi')"
 
         raises(ValueError, self.array('i').tounicode)
-        assert self.array('u', unicode('hello')).tounicode() == \
-               unicode('hello')
+        assert self.array('u', u'hello').tounicode() == u'hello'
 
     def test_empty_tostring(self):
         a = self.array('l')
@@ -493,14 +479,14 @@
 
     def test_compare(self):
         class comparable(object):
-            def __cmp__(self, other):
-                return 0
+            def __eq__(self, other):
+                return True
         class incomparable(object):
             pass
 
         for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'),
                          ('abc', 'acb', 'c'),
-                         (unicode('abc'), unicode('acb'), 'u')):
+                         (u'abc', u'acb', 'u')):
             for t in tt:
                 a = self.array(t, v1)
                 b = self.array(t, v1)
@@ -767,16 +753,16 @@
                 self.height = height
                 return self
 
-            def _index(self, (x,y)):
+            def _index(self, x, y):
                 x = min(max(x, 0), self.width-1)
                 y = min(max(y, 0), self.height-1)
                 return y * self.width + x
 
             def __getitem__(self, i):
-                return array.__getitem__(self, self._index(i))
+                return array.__getitem__(self, self._index(*i))
 
             def __setitem__(self, i, val):
-                return array.__setitem__(self, self._index(i), val)
+                return array.__setitem__(self, self._index(*i), val)
 
         img = Image(5, 10, 'B')
         for y in range(10):
@@ -844,8 +830,8 @@
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
     def test_unicode_outofrange(self):
-        a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
-        b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
+        a = self.array('u', u'\x01\u263a\x00\ufeff')
+        b = self.array('u', u'\x01\u263a\x00\ufeff')
         b.byteswap()
         assert a != b
 
@@ -853,7 +839,7 @@
         import sys
         if sys.maxunicode == 0xffff:
             skip("test for 32-bit unicodes")
-        a = self.array('u', '\xff\xff\xff\xff')
+        a = self.array('u', b'\xff\xff\xff\xff')
         assert len(a) == 1
         assert repr(a[0]) == "u'\Uffffffff'"
         if sys.maxint == 2147483647:
@@ -954,28 +940,6 @@
         assert a[0] == u'b'
 
 
-class TestCPythonsOwnArray(BaseArrayTests):
-    def setup_class(cls):
-        import array
-        cls.array = array.array
-        import struct
-        cls.struct = struct
-        cls.tempfile = str(pytest.ensuretemp('array').join('tmpfile'))
-        cls.maxint = sys.maxint
-
-
-class AppTestArray(BaseArrayTests):
-    spaceconfig = {'usemodules': ['array', 'struct', '_rawffi', 'binascii']}
-
-    def setup_class(cls):
-        cls.w_array = cls.space.appexec([], """():
-            import array
-            return array.array
-        """)
-        cls.w_tempfile = cls.space.wrap(
-            str(pytest.ensuretemp('array').join('tmpfile')))
-        cls.w_maxint = cls.space.wrap(sys.maxint)
-
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/array/test/test_array_old.py b/pypy/module/array/test/test_array_old.py
deleted file mode 100644
--- a/pypy/module/array/test/test_array_old.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# minimal tests.  See also lib-python/modified-2.4.1/test/test_array.
-
-import py
-from py.test import raises
-import struct
-
-
-class BaseArrayTests:
-    # XXX very incomplete
-
-    native_sizes = {'l': struct.calcsize('l')}
-
-    def test_attributes(self):
-        a = self.array.array('c')
-        assert a.typecode == 'c'
-        assert a.itemsize == 1
-        a = self.array.array('l')
-        assert a.typecode == 'l'
-        assert a.itemsize == self.native_sizes['l']
-
-    def test_imul(self):
-        a = self.array.array('i', [12, 34])
-        a *= 3
-        assert a.tolist() == [12, 34] * 3
-
-    def test_unicode(self):
-        a = self.array.array('u')
-        a.fromunicode(unichr(9999))
-        assert len(a) == 1
-        assert a.tolist() == [unichr(9999)]
-
-    def test_pickle(self):
-        import sys
-        if sys.version_info < (2, 5):
-            py.test.skip("array.array not picklable before python 2.5")
-        import pickle
-
-        for content in [[56, -12, 34], []]:
-            a = self.array.array('i', content)
-            a2 = pickle.loads(pickle.dumps(a))
-            assert type(a2) is self.array.array
-            assert list(a2) == content
-
-    def test_init_vs_new(self):
-        import sys
-        if sys.version_info < (2, 5):
-            py.test.skip("array.array constructor changed in 2.5")
-        class A(self.array.array):
-            def __init__(self, *args, **kwds):
-                self.args = args
-                self.kwds = kwds
-
-        a = A('c', foo='bar')
-        assert a.args == ('c',)
-        assert a.kwds == {'foo': 'bar'}
-        a = A('i', range(10), some=42)
-        assert a.args == ('i', range(10))
-        assert a.kwds == {'some': 42}
-        raises(TypeError, A)
-        raises(TypeError, A, 42)
-        raises(TypeError, A, 'i', [], [])
-        raises(TypeError, self.array.array, 'i', [], foo='bar')
-
-
-class TestCPythonsOwnArray(BaseArrayTests):
-
-    def setup_class(cls):
-        import array
-        cls.array = array
-
-
-## class TestArrayOnTopOfCPython(BaseArrayTests):
-
-##     def setup_class(cls):
-##         from pypy.tool.lib_pypy import LIB_PYPY
-##         if not hasattr(struct, 'pack_into'):
-##             py.test.skip("requires CPython >= 2.5")
-##         import new
-##         path = LIB_PYPY.join('array.py')
-##         myarraymodule = new.module('array')
-##         execfile(str(path), myarraymodule.__dict__)
-##         cls.array = myarraymodule
-
-##     def test_unicode(self):
-##         py.test.skip("no 'u' type code in CPython's struct module")
-
-##     def test_pickle(self):
-##         py.test.skip("pickle getting confused by the hack in setup_class()")
-
-
-class AppTestArray(BaseArrayTests):
-    spaceconfig = {'usemodules': ['struct', 'array', 'binascii']}
-
-    def setup_class(cls):
-        """Import the array module and make it available as self.array."""
-        cls.w_array = cls.space.getbuiltinmodule('array')
-        cls.w_native_sizes = cls.space.wrap(cls.native_sizes)
-
-
-## class AppTestArrayWithRawFFI(AppTestArray):
-##     """
-##     The same as the base class, but with a space that also includes the
-##     _rawffi module.  The array module internally uses it in this case.
-##     """
-##     spaceconfig = dict(usemodules=['struct', '_rawffi'])
-
-##     def test_buffer_info(self):
-##         a = self.array.array('l', [123, 456])
-##         assert a.itemsize == self.native_sizes['l']
-##         address, length = a.buffer_info()
-##         assert length == 2      # and not 2 * self.native_sizes['l']
-##         assert address != 0
-##         # should check the address via some unsafe peeking, but it's
-##         # not easy on top of py.py
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -79,11 +79,16 @@
 CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
                                        hints={'nolength': True}),
                           use_cache=False)
+CONST_STRINGP = lltype.Ptr(lltype.Array(rffi.CCHARP,
+                                       hints={'nolength': True}),
+                          use_cache=False)
 CONST_WSTRING = lltype.Ptr(lltype.Array(lltype.UniChar,
                                         hints={'nolength': True}),
                            use_cache=False)
 assert CONST_STRING is not rffi.CCHARP
 assert CONST_STRING == rffi.CCHARP
+assert CONST_STRINGP is not rffi.CCHARPP
+assert CONST_STRINGP == rffi.CCHARPP
 assert CONST_WSTRING is not rffi.CWCHARP
 assert CONST_WSTRING == rffi.CWCHARP
 
@@ -1004,6 +1009,8 @@
     for i, argtype in enumerate(func.argtypes):
         if argtype is CONST_STRING:
             arg = 'const char *@'
+        elif argtype is CONST_STRINGP:
+            arg = 'const char **@'
         elif argtype is CONST_WSTRING:
             arg = 'const wchar_t *@'
         else:
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -2,7 +2,6 @@
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
 from pypy.module.cpyext.pyobject import PyObject
-from pypy.module.cpyext.bytesobject import PyBytesObject
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_CheckBuffer(space, pyobj):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,6 +1,6 @@
 from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
                          Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref, from_ref
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import widen
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -132,11 +132,22 @@
 def PyMemoryView_FromObject(space, w_obj):
     return space.call_method(space.builtin, "memoryview", w_obj)
 
+ at cpython_api([lltype.Ptr(Py_buffer)], PyObject)
+def PyMemoryView_FromBuffer(space, view):
+    """Create a memoryview object wrapping the given buffer-info structure view.
+    The memoryview object then owns the buffer, which means you shouldn't
+    try to release it yourself: it will be released on deallocation of the
+    memoryview object."""
+    w_obj = from_ref(space, view.c_obj)
+    if isinstance(w_obj, W_MemoryView):
+        return w_obj
+    return space.call_method(space.builtin, "memoryview", w_obj)
+
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
     # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
     # XXX needed for numpy on py3k
-    raise NotImplementedError('PyMemoryView_GET_BUFFER')
+    raise NotImplementedError('PyMemoryView_GET_BASE')
 
 @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
 def PyMemoryView_GET_BUFFER(space, w_obj):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -311,7 +311,7 @@
 def PyClassMethod_New(space, w_func):
     return space.wrap(ClassMethod(w_func))
 
- at cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
+ at cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
 def PyDescr_NewMethod(space, w_type, method):
     return space.wrap(W_PyCMethodObject(space, method, w_type))
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -3,7 +3,7 @@
     cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
     PyVarObject, Py_buffer, size_t,
     Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
-    Py_GE, CONST_STRING, FILEP, fwrite)
+    Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
     get_typedescr, _Py_NewReference)
@@ -429,7 +429,7 @@
     is active then NULL is returned but PyErr_Occurred() will return false."""
     return space.call_function(space.builtin.get('dir'), w_o)
 
- at cpython_api([PyObject, rffi.CCHARPP, Py_ssize_tP], rffi.INT_real, error=-1)
+ at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
 def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
     """Returns a pointer to a read-only memory location usable as
     character-based input.  The obj argument must support the single-segment
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py

From pypy.commits at gmail.com  Wed Nov  2 09:39:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 06:39:25 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: vector_ext object should never
 be None, move it to abstract base class (it is enabled at runtime)
Message-ID: <5819ec8d.88571c0a.fa342.744b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88081:58c1cfa77a55
Date: 2016-11-02 14:38 +0100
http://bitbucket.org/pypy/pypy/changeset/58c1cfa77a55/

Log:	vector_ext object should never be None, move it to abstract base
	class (it is enabled at runtime)

diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -22,6 +22,8 @@
     with_threads = False
     frame_reg = regloc.ebp
 
+    vector_ext = X86VectorExt()
+
     # can an ISA instruction handle a factor to the offset?
     load_supported_factors = (1,2,4,8)
 
@@ -146,7 +148,6 @@
     supports_longlong = False
 
 class CPU_X86_64(AbstractX86CPU):
-    vector_ext = X86VectorExt()
 
     backend_name = 'x86_64'
     NUM_REGS = 16

From pypy.commits at gmail.com  Wed Nov  2 10:10:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 07:10:06 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: revert last checkin,
 check if vector_ext is not None
Message-ID: <5819f3be.87101c0a.e4d70.82a1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88082:f9fb13244302
Date: 2016-11-02 15:09 +0100
http://bitbucket.org/pypy/pypy/changeset/f9fb13244302/

Log:	revert last checkin, check if vector_ext is not None

diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -22,7 +22,7 @@
     with_threads = False
     frame_reg = regloc.ebp
 
-    vector_ext = X86VectorExt()
+    vector_ext = None
 
     # can an ISA instruction handle a factor to the offset?
     load_supported_factors = (1,2,4,8)
@@ -148,7 +148,7 @@
     supports_longlong = False
 
 class CPU_X86_64(AbstractX86CPU):
-
+    vector_ext = X86VectorExt()
     backend_name = 'x86_64'
     NUM_REGS = 16
     CALLEE_SAVE_REGISTERS = [regloc.ebx, regloc.r12, regloc.r13, regloc.r14, regloc.r15]
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -302,7 +302,8 @@
         history.cut(cut_at)
         return None
 
-    if ((warmstate.vec and jitdriver_sd.vec) or warmstate.vec_all) and metainterp.cpu.vector_ext.is_enabled():
+    if ((warmstate.vec and jitdriver_sd.vec) or warmstate.vec_all) and \
+        metainterp.cpu.vector_ext and metainterp.cpu.vector_ext.is_enabled():
         from rpython.jit.metainterp.optimizeopt.vector import optimize_vector
         loop_info, loop_ops = optimize_vector(trace, metainterp_sd,
                                               jitdriver_sd, warmstate,

From pypy.commits at gmail.com  Wed Nov  2 10:35:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 07:35:59 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: assert that vector_ext cannot
 be none at that position (optmize_vector will never be entered if is none)
Message-ID: <5819f9cf.cb091c0a.8841a.84e7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88083:12b46db4a793
Date: 2016-11-02 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/12b46db4a793/

Log:	assert that vector_ext cannot be none at that position
	(optmize_vector will never be entered if is none)

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -77,7 +77,9 @@
         assert isinstance(arg, VectorOp)
         size = arg.bytesize
         temp = X86_64_XMM_SCRATCH_REG
-        load = arg.bytesize * arg.count - self.cpu.vector_ext.register_size
+        ve = self.cpu.vector_ext
+        assert ve is not None # MUST hold, optimize_vector is never entered if vector_ext is entered
+        load = arg.bytesize * arg.count - ve.register_size
         assert load <= 0
         if true:
             self.mc.PXOR(temp, temp)

From pypy.commits at gmail.com  Wed Nov  2 11:32:05 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 08:32:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix implementation-specific test
Message-ID: <581a06f5.0209c20a.36aab.12ef@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88084:d58a4a7ab7d4
Date: 2016-11-02 15:31 +0000
http://bitbucket.org/pypy/pypy/changeset/d58a4a7ab7d4/

Log:	Fix implementation-specific test

diff --git a/lib-python/3/test/test_collections.py b/lib-python/3/test/test_collections.py
--- a/lib-python/3/test/test_collections.py
+++ b/lib-python/3/test/test_collections.py
@@ -13,6 +13,7 @@
 import re
 import sys
 import types
+import gc
 from collections import UserDict, UserString, UserList
 from collections import ChainMap
 from collections import deque
@@ -2052,7 +2053,9 @@
             obj = MyOD([(None, obj)])
             obj.i = i
         del obj
-        support.gc_collect()
+        # PyPy change: we only collect 1 MyOD instance per GC
+        for _ in range(100):
+            gc.collect()
         self.assertEqual(deleted, list(reversed(range(100))))
 
     def test_delitem_hash_collision(self):

From pypy.commits at gmail.com  Wed Nov  2 11:51:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Nov 2016 08:51:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: #2408: support for os.utime('foo',
 (-12.34, -56.78))
Message-ID: <581a0b6a.0726c20a.20839.16da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88085:8822b3e22aac
Date: 2016-11-02 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/8822b3e22aac/

Log:	#2408: support for os.utime('foo', (-12.34, -56.78))

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1362,8 +1362,14 @@
 def _time_to_timeval(t, l_timeval):
     import math
     fracpart, intpart = math.modf(t)
-    rffi.setintfield(l_timeval, 'c_tv_sec', int(intpart))
-    rffi.setintfield(l_timeval, 'c_tv_usec', int(fracpart * 1e6))
+    intpart = int(intpart)
+    fracpart = int(fracpart * 1e6)
+    if fracpart < 0:
+        intpart -= 1
+        fracpart += 1000000
+    assert 0 <= fracpart < 1000000
+    rffi.setintfield(l_timeval, 'c_tv_sec', intpart)
+    rffi.setintfield(l_timeval, 'c_tv_usec', fracpart)
 
 if not _WIN32:
     TMSP = lltype.Ptr(TMS)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -59,6 +59,17 @@
         compile(f, (str, float))(str(fname), t1)
         assert t1 == os.stat(str(fname)).st_mtime
 
+    def test_utime_negative_fraction(self):
+        def f(fname, t1):
+            os.utime(fname, (t1, t1))
+
+        fname = udir.join('test_utime_negative_fraction.txt')
+        fname.ensure()
+        t1 = -123.75
+        compile(f, (str, float))(str(fname), t1)
+        got = os.stat(str(fname)).st_mtime
+        assert got == -123 or got == -123.75
+
     @win_only
     def test__getfullpathname(self):
         posix = __import__(os.name)

From pypy.commits at gmail.com  Wed Nov  2 11:56:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 02 Nov 2016 08:56:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-ssl: renamed library in lib_pypy,
 copied ssl.py from cpython's Lib directory,
 checkin the exposed version _ssl fo the ssl module
Message-ID: <581a0cc8.4a6ec20a.b8d27.1bc0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88086:278636f9c9c0
Date: 2016-11-02 16:56 +0100
http://bitbucket.org/pypy/pypy/changeset/278636f9c9c0/

Log:	renamed library in lib_pypy, copied ssl.py from cpython's Lib
	directory, checkin the exposed version _ssl fo the ssl module

diff too long, truncating to 2000 out of 16480 lines

diff --git a/lib_pypy/_ssl/.gitignore b/lib_pypy/_ssl/.gitignore
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Vim
+
+*.swp
+*.swo
diff --git a/lib_pypy/_ssl/LICENSE b/lib_pypy/_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_ssl/README.md b/lib_pypy/_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/README.md
@@ -0,0 +1,4 @@
+# PyPy's SSL module
+
+Most of the CFFI code is copied from cryptography
+
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/__init__.py
@@ -0,0 +1,3 @@
+from _ssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION,
+        _test_decode_cert, _SSLContext)
+from _ssl._stdssl import *
diff --git a/lib_pypy/_ssl/_cffi_src/__init__.py b/lib_pypy/_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_ssl/_cffi_src/build_openssl.py b/lib_pypy/_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,85 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from openssl._cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_ssl/_cffi_src/build_padding.py b/lib_pypy/_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;
+
+typedef enum {
+    kSSLSessionOptionBreakOnServerAuth,
+    kSSLSessionOptionBreakOnCertRequested,
+} SSLSessionOption;
+
+typedef enum {
+    kNeverAuthenticate,
+    kAlwaysAuthenticate,
+    kTryAuthenticate
+} SSLAuthenticate;
+
+typedef enum {
+    kSSLIdle,
+    kSSLHandshake,
+    kSSLConnected,
+    kSSLClosed,
+    kSSLAborted
+} SSLSessionState;
+
+typedef enum {
+    kSSLProtocolUnknown = 0,
+    kSSLProtocol3       = 2,
+    kTLSProtocol1       = 4,
+    /* DEPRECATED on iOS */
+    kSSLProtocol2       = 1,
+    kSSLProtocol3Only   = 3,
+    kTLSProtocol1Only   = 5,
+    kSSLProtocolAll     = 6,
+} SSLProtocol;
+
+typedef UInt32 SSLCipherSuite;
+enum {
+   SSL_NULL_WITH_NULL_NULL =               0x0000,
+   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
+   SSL_RSA_WITH_NULL_SHA =                 0x0002,
+   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
+   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
+   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
+   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
+   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
+   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
+   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
+   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
+   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
+   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
+   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
+   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
+   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
+   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
+   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
+   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
+   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
+   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
+   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
+   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
+   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
+   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
+   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
+   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
+   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
+   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
+   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
+
+   /* TLS addenda using AES, per RFC 3268 */
+   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
+   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
+
+   /* ECDSA addenda, RFC 4492 */
+   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
+   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
+   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
+   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
+   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
+   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
+   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
+   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
+   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
+   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
+   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
+   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
+   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
+   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
+   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
+   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
+   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
+
+   /* TLS 1.2 addenda, RFC 5246 */
+   /* Initial state. */
+   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
+
+   /* Server provided RSA certificate for key exchange. */
+   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
+   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
+   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
+   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
+   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
+   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
+   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
+   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
+
+   /* Server-authenticated (and optionally client-authenticated)
+      Diffie-Hellman. */
+   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
+   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
+   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
+   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
+
+   /* Completely anonymous Diffie-Hellman */
+   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
+   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
+
+   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
+      for TLS. */
+   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
+   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
+   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
+   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
+   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
+   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
+   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
+   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
+   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
+   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
+   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
+   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      HMAC SHA-256/384. */
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      SHA-256/384 and AES Galois Counter Mode (GCM) */
+   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
+   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
+   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
+   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
+   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
+   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
+   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
+   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
+
+   /* RFC 5746 - Secure Renegotiation */
+   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
+
+   /*
+   * Tags for SSL 2 cipher kinds that are not specified
+   * for SSL 3.
+   */
+   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
+   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
+   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
+   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
+   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
+};
+
+typedef enum {
+    kSSLClientCertNone,
+    kSSLClientCertRequested,
+    kSSLClientCertSent,
+    kSSLClientCertRejected
+} SSLClientCertificateState;
+
+enum {
+    errSSLProtocol              = -9800,
+    errSSLNegotiation           = -9801,
+    errSSLFatalAlert            = -9802,
+    errSSLWouldBlock            = -9803,
+    errSSLSessionNotFound       = -9804,
+    errSSLClosedGraceful        = -9805,
+    errSSLClosedAbort           = -9806,
+    errSSLXCertChainInvalid     = -9807,
+    errSSLBadCert               = -9808,
+    errSSLCrypto                = -9809,
+    errSSLInternal              = -9810,
+    errSSLModuleAttach          = -9811,
+    errSSLUnknownRootCert       = -9812,
+    errSSLNoRootCert            = -9813,
+    errSSLCertExpired           = -9814,
+    errSSLCertNotYetValid       = -9815,
+    errSSLClosedNoNotify        = -9816,
+    errSSLBufferOverflow        = -9817,
+    errSSLBadCipherSuite        = -9818,
+    errSSLPeerUnexpectedMsg     = -9819,
+    errSSLPeerBadRecordMac      = -9820,
+    errSSLPeerDecryptionFail    = -9821,
+    errSSLPeerRecordOverflow    = -9822,
+    errSSLPeerDecompressFail    = -9823,
+    errSSLPeerHandshakeFail     = -9824,
+    errSSLPeerBadCert           = -9825,
+    errSSLPeerUnsupportedCert   = -9826,
+    errSSLPeerCertRevoked       = -9827,
+    errSSLPeerCertExpired       = -9828,
+    errSSLPeerCertUnknown       = -9829,
+    errSSLIllegalParam          = -9830,
+    errSSLPeerUnknownCA         = -9831,
+    errSSLPeerAccessDenied      = -9832,
+    errSSLPeerDecodeError       = -9833,
+    errSSLPeerDecryptError      = -9834,
+    errSSLPeerExportRestriction = -9835,
+    errSSLPeerProtocolVersion   = -9836,
+    errSSLPeerInsufficientSecurity = -9837,
+    errSSLPeerInternalError     = -9838,
+    errSSLPeerUserCancelled     = -9839,
+    errSSLPeerNoRenegotiation   = -9840,
+    errSSLServerAuthCompleted   = -9841,
+    errSSLClientCertRequested   = -9842,
+    errSSLHostNameMismatch      = -9843,
+    errSSLConnectionRefused     = -9844,
+    errSSLDecryptionFail        = -9845,
+    errSSLBadRecordMac          = -9846,
+    errSSLRecordOverflow        = -9847,
+    errSSLBadConfiguration      = -9848,
+    errSSLLast                  = -9849     /* end of range, to be deleted */
+};
+"""
+
+FUNCTIONS = """
+OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
+OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
+OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
+OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
+
+OSStatus SSLHandshake(SSLContextRef);
+OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
+OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
+OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
+OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
+OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
+OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
+OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
+OSStatus SSLClose(SSLContextRef);
+
+OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
+OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
+OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
+OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
+
+OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
+OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
+OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
+OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
+OSStatus SSLGetClientCertificateState(SSLContextRef,
+                                      SSLClientCertificateState *);
+OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
+
+OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
+OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
+OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c
@@ -0,0 +1,22 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
+                                            uint8_t *b, size_t len_b) {
+    size_t i = 0;
+    uint8_t mismatch = 0;
+    if (len_a != len_b) {
+        return 0;
+    }
+    for (i = 0; i < len_a; i++) {
+        mismatch |= a[i] ^ b[i];
+    }
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
+                                            size_t);
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c
@@ -0,0 +1,63 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+/* Returns the value of the input with the most-significant-bit copied to all
+   of the bits. */
+static uint8_t Cryptography_DUPLICATE_MSB_TO_ALL(uint8_t a) {
+    return (1 - (a >> (sizeof(uint8_t) * 8 - 1))) - 1;
+}
+
+/* This returns 0xFF if a < b else 0x00, but does so in a constant time
+   fashion */
+static uint8_t Cryptography_constant_time_lt(uint8_t a, uint8_t b) {
+    a -= b;
+    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
+}
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
+                                         uint8_t block_len) {
+    uint8_t i;
+    uint8_t pad_size = data[block_len - 1];
+    uint8_t mismatch = 0;
+    for (i = 0; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint8_t b = data[block_len - 1 - i];
+        mismatch |= (mask & (pad_size ^ b));
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
+
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
+                                            uint8_t block_len) {
+    uint8_t i;
+    uint8_t pad_size = data[block_len - 1];
+    uint8_t mismatch = 0;
+    /* Skip the first one with the pad size */
+    for (i = 1; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint8_t b = data[block_len - 1 - i];
+        mismatch |= (mask & b);
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_ssl/_cffi_src/openssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/openssl/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_ssl/_cffi_src/openssl/aes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/openssl/aes.py
@@ -0,0 +1,50 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/aes.h>
+"""
+
+TYPES = """
+static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
+
+struct aes_key_st {
+    ...;
+};
+typedef struct aes_key_st AES_KEY;
+"""
+
+FUNCTIONS = """
+int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
+int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
+
+int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                 const unsigned char *, unsigned int);
+int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                   const unsigned char *, unsigned int);
+"""
+
+MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_ssl/_cffi_src/openssl/asn1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/openssl/asn1.py
@@ -0,0 +1,164 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/asn1.h>
+"""
+
+TYPES = """
+typedef int... time_t;
+
+typedef int ASN1_BOOLEAN;
+typedef ... ASN1_INTEGER;
+
+struct asn1_string_st {
+    int length;
+    int type;
+    unsigned char *data;
+    long flags;
+};
+
+typedef struct asn1_string_st ASN1_OCTET_STRING;
+typedef struct asn1_string_st ASN1_IA5STRING;
+typedef struct asn1_string_st ASN1_BIT_STRING;
+typedef struct asn1_string_st ASN1_TIME;
+typedef ... ASN1_OBJECT;
+typedef struct asn1_string_st ASN1_STRING;
+typedef struct asn1_string_st ASN1_UTF8STRING;
+typedef ... ASN1_TYPE;
+typedef ... ASN1_GENERALIZEDTIME;
+typedef ... ASN1_ENUMERATED;
+typedef ... ASN1_ITEM;
+typedef ... ASN1_VALUE;
+
+typedef ... ASN1_ITEM_EXP;
+
+typedef ... ASN1_UTCTIME;
+
+static const int V_ASN1_GENERALIZEDTIME;
+
+static const int MBSTRING_FLAG;
+static const int MBSTRING_ASC;
+static const int MBSTRING_BMP;
+static const int MBSTRING_UTF8;
+static const int MBSTRING_UNIV;
+"""
+
+FUNCTIONS = """
+ASN1_OBJECT *ASN1_OBJECT_new(void);
+void ASN1_OBJECT_free(ASN1_OBJECT *);
+
+/*  ASN1 OBJECT IDENTIFIER */
+ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
+
+/*  ASN1 STRING */
+ASN1_STRING *ASN1_STRING_new(void);
+ASN1_STRING *ASN1_STRING_type_new(int);
+void ASN1_STRING_free(ASN1_STRING *);
+unsigned char *ASN1_STRING_data(ASN1_STRING *);
+int ASN1_STRING_set(ASN1_STRING *, const void *, int);
+
+/*  ASN1 OCTET STRING */
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
+void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
+
+/* ASN1 IA5STRING */
+ASN1_IA5STRING *ASN1_IA5STRING_new(void);
+
+/*  ASN1 INTEGER */
+ASN1_INTEGER *ASN1_INTEGER_new(void);
+void ASN1_INTEGER_free(ASN1_INTEGER *);
+int ASN1_INTEGER_set(ASN1_INTEGER *, long);
+
+/*  ASN1 TIME */
+ASN1_TIME *ASN1_TIME_new(void);
+void ASN1_TIME_free(ASN1_TIME *);
+ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
+                                                   ASN1_GENERALIZEDTIME **);
+ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
+
+/*  ASN1 UTCTIME */
+ASN1_UTCTIME *ASN1_UTCTIME_new(void);
+void ASN1_UTCTIME_free(ASN1_UTCTIME *);
+int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
+ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
+
+/*  ASN1 GENERALIZEDTIME */
+int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
+ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
+void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
+int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
+
+/*  ASN1 ENUMERATED */
+ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
+void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
+int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
+
+ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
+                          const ASN1_ITEM *);
+int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
+"""
+
+MACROS = """
+/* These became const ASN1_* in 1.1.0 */
+int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
+int ASN1_STRING_type(ASN1_STRING *);
+int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
+long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
+int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
+
+ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
+void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
+
+ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
+void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
+int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
+int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
+int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
+/* This is not a macro, but is const on some versions of OpenSSL */
+int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
+ASN1_TIME *M_ASN1_TIME_dup(void *);
+const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
+
+/* These aren't macros these arguments are all const X on openssl > 1.0.x */
+
+int ASN1_TIME_print(BIO *, ASN1_TIME *);
+int ASN1_STRING_length(ASN1_STRING *);
+ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
+int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
+int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
+
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
+
+ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
+int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
+long ASN1_INTEGER_get(ASN1_INTEGER *);
+
+BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
+ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
+
+/* These isn't a macro the arg is const on openssl 1.0.2+ */
+int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
+int ASN1_UTCTIME_check(ASN1_UTCTIME *);
+
+/* Not a macro, const on openssl 1.0 */
+int ASN1_STRING_set_default_mask_asc(char *);
+
+int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
+ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
+"""
+
+CUSTOMIZATIONS = """
+/* This macro is removed in 1.1.0. We re-add it if required to support
+   pyOpenSSL versions older than whatever resolves
+   https://github.com/pyca/pyopenssl/issues/431 */
+#if !defined(M_ASN1_TIME_dup)
+#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
+#endif
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_ssl/_cffi_src/openssl/bignum.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/openssl/bignum.py
@@ -0,0 +1,88 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bn.h>
+"""
+
+TYPES = """
+typedef ... BN_CTX;
+typedef ... BIGNUM;
+typedef int... BN_ULONG;
+"""
+
+FUNCTIONS = """
+BIGNUM *BN_new(void);
+void BN_free(BIGNUM *);
+
+BN_CTX *BN_CTX_new(void);
+void BN_CTX_free(BN_CTX *);
+
+void BN_CTX_start(BN_CTX *);
+BIGNUM *BN_CTX_get(BN_CTX *);
+void BN_CTX_end(BN_CTX *);
+
+BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
+BIGNUM *BN_dup(const BIGNUM *);
+
+int BN_set_word(BIGNUM *, BN_ULONG);
+BN_ULONG BN_get_word(const BIGNUM *);
+
+const BIGNUM *BN_value_one(void);
+
+char *BN_bn2hex(const BIGNUM *);
+int BN_hex2bn(BIGNUM **, const char *);
+int BN_dec2bn(BIGNUM **, const char *);
+
+int BN_bn2bin(const BIGNUM *, unsigned char *);
+BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
+
+int BN_num_bits(const BIGNUM *);
+
+int BN_cmp(const BIGNUM *, const BIGNUM *);
+int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_set_bit(BIGNUM *, int);
+int BN_clear_bit(BIGNUM *, int);
+
+int BN_is_bit_set(const BIGNUM *, int);
+
+int BN_mask_bits(BIGNUM *, int);
+"""
+
+MACROS = """
+int BN_num_bytes(const BIGNUM *);
+
+int BN_zero(BIGNUM *);
+int BN_one(BIGNUM *);
+int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_lshift(BIGNUM *, const BIGNUM *, int);
+int BN_lshift1(BIGNUM *, BIGNUM *);
+
+int BN_rshift(BIGNUM *, BIGNUM *, int);
+int BN_rshift1(BIGNUM *, BIGNUM *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_ssl/_cffi_src/openssl/bio.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_ssl/_cffi_src/openssl/bio.py
@@ -0,0 +1,136 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bio.h>
+"""
+
+TYPES = """
+typedef struct bio_st BIO;
+typedef void bio_info_cb(BIO *, int, const char *, int, long, long);
+typedef ... bio_st;
+typedef ... BIO_METHOD;
+typedef ... BUF_MEM;
+
+static const int BIO_TYPE_MEM;
+static const int BIO_TYPE_FILE;
+static const int BIO_TYPE_FD;
+static const int BIO_TYPE_SOCKET;
+static const int BIO_TYPE_CONNECT;
+static const int BIO_TYPE_ACCEPT;
+static const int BIO_TYPE_NULL;
+static const int BIO_CLOSE;
+static const int BIO_NOCLOSE;
+static const int BIO_TYPE_SOURCE_SINK;
+static const int BIO_CTRL_RESET;
+static const int BIO_CTRL_EOF;
+static const int BIO_CTRL_SET;
+static const int BIO_CTRL_SET_CLOSE;
+static const int BIO_CTRL_FLUSH;
+static const int BIO_CTRL_DUP;
+static const int BIO_CTRL_GET_CLOSE;
+static const int BIO_CTRL_INFO;
+static const int BIO_CTRL_GET;
+static const int BIO_CTRL_PENDING;
+static const int BIO_CTRL_WPENDING;
+static const int BIO_C_FILE_SEEK;
+static const int BIO_C_FILE_TELL;
+static const int BIO_TYPE_NONE;
+static const int BIO_TYPE_NBIO_TEST;
+static const int BIO_TYPE_BIO;
+static const int BIO_TYPE_DESCRIPTOR;
+static const int BIO_FLAGS_READ;
+static const int BIO_FLAGS_WRITE;
+static const int BIO_FLAGS_IO_SPECIAL;
+static const int BIO_FLAGS_RWS;
+static const int BIO_FLAGS_SHOULD_RETRY;
+static const int BIO_TYPE_NULL_FILTER;
+static const int BIO_TYPE_SSL;
+static const int BIO_TYPE_MD;
+static const int BIO_TYPE_BUFFER;
+static const int BIO_TYPE_CIPHER;
+static const int BIO_TYPE_BASE64;
+static const int BIO_TYPE_FILTER;
+"""
+
+FUNCTIONS = """
+int BIO_free(BIO *);
+void BIO_vfree(BIO *);
+void BIO_free_all(BIO *);
+BIO *BIO_push(BIO *, BIO *);
+BIO *BIO_pop(BIO *);
+BIO *BIO_next(BIO *);
+BIO *BIO_find_type(BIO *, int);
+BIO *BIO_new_file(const char *, const char *);
+BIO *BIO_new_fp(FILE *, int);
+BIO *BIO_new_fd(int, int);
+BIO *BIO_new_socket(int, int);
+long BIO_ctrl(BIO *, int, long, void *);
+long BIO_callback_ctrl(
+    BIO *,
+    int,
+    void (*)(struct bio_st *, int, const char *, int, long, long)
+);
+long BIO_int_ctrl(BIO *, int, long, int);
+size_t BIO_ctrl_pending(BIO *);
+size_t BIO_ctrl_wpending(BIO *);
+int BIO_read(BIO *, void *, int);
+int BIO_gets(BIO *, char *, int);
+int BIO_write(BIO *, const void *, int);
+int BIO_puts(BIO *, const char *);
+int BIO_method_type(const BIO *);
+"""
+
+MACROS = """
+/* These added const to BIO_METHOD in 1.1.0 */
+BIO *BIO_new(BIO_METHOD *);
+BIO_METHOD *BIO_s_mem(void);
+BIO_METHOD *BIO_s_file(void);
+BIO_METHOD *BIO_s_fd(void);
+BIO_METHOD *BIO_s_socket(void);
+BIO_METHOD *BIO_s_null(void);
+BIO_METHOD *BIO_f_null(void);
+BIO_METHOD *BIO_f_buffer(void);
+/* BIO_new_mem_buf became const void * in 1.0.2g */
+BIO *BIO_new_mem_buf(void *, int);
+long BIO_set_fd(BIO *, long, int);
+long BIO_get_fd(BIO *, char *);
+long BIO_set_mem_eof_return(BIO *, int);
+long BIO_get_mem_data(BIO *, char **);
+long BIO_set_mem_buf(BIO *, BUF_MEM *, int);
+long BIO_get_mem_ptr(BIO *, BUF_MEM **);
+long BIO_set_fp(BIO *, FILE *, int);
+long BIO_get_fp(BIO *, FILE **);
+long BIO_read_filename(BIO *, char *);
+long BIO_write_filename(BIO *, char *);
+long BIO_append_filename(BIO *, char *);
+long BIO_rw_filename(BIO *, char *);
+int BIO_should_read(BIO *);
+int BIO_should_write(BIO *);
+int BIO_should_io_special(BIO *);
+int BIO_retry_type(BIO *);
+int BIO_should_retry(BIO *);
+int BIO_reset(BIO *);
+int BIO_seek(BIO *, int);
+int BIO_tell(BIO *);
+int BIO_flush(BIO *);
+int BIO_eof(BIO *);

From pypy.commits at gmail.com  Wed Nov  2 13:07:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 10:07:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip the tests for
 _collections.OrderedDict if we don't have it (and ATM we don't)
Message-ID: <581a1d47.8ca71c0a.ff099.ce14@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88087:f655e46d5ffb
Date: 2016-11-02 17:06 +0000
http://bitbucket.org/pypy/pypy/changeset/f655e46d5ffb/

Log:	Skip the tests for _collections.OrderedDict if we don't have it (and
	ATM we don't)

diff --git a/lib-python/3/test/test_collections.py b/lib-python/3/test/test_collections.py
--- a/lib-python/3/test/test_collections.py
+++ b/lib-python/3/test/test_collections.py
@@ -1631,7 +1631,7 @@
 ################################################################################
 
 py_coll = import_fresh_module('collections', blocked=['_collections'])
-c_coll = import_fresh_module('collections', fresh=['_collections'])
+c_coll = import_fresh_module('_collections', fresh=['_collections'])
 
 
 @contextlib.contextmanager
@@ -2215,11 +2215,11 @@
     OrderedDict = py_coll.OrderedDict
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase):
 
     module = c_coll
-    OrderedDict = c_coll.OrderedDict
+    OrderedDict = getattr(c_coll, 'OrderedDict', None)
 
     def test_key_change_during_iteration(self):
         OrderedDict = self.OrderedDict
@@ -2249,8 +2249,10 @@
 class CPythonOrderedDictSubclassTests(CPythonOrderedDictTests):
 
     module = c_coll
-    class OrderedDict(c_coll.OrderedDict):
-        pass
+
+    if hasattr(c_coll, 'OrderedDict'):
+        class OrderedDict(c_coll.OrderedDict):
+            pass
 
 
 class PurePythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
@@ -2264,7 +2266,7 @@
         self.assertRaises(KeyError, d.popitem)
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
 
     @classmethod
@@ -2289,7 +2291,7 @@
         self.assertRaises(KeyError, d.popitem)
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonSubclassMappingTests(mapping_tests.BasicTestMappingProtocol):
 
     @classmethod

From pypy.commits at gmail.com  Wed Nov  2 13:39:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 10:39:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: dict should not have a __reversed__()
 method
Message-ID: <581a24ed.ca73c20a.b998c.5100@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88088:46e7554526ef
Date: 2016-11-02 17:39 +0000
http://bitbucket.org/pypy/pypy/changeset/46e7554526ef/

Log:	dict should not have a __reversed__() method

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -204,10 +204,6 @@
         except KeyError:
             space.raise_key_error(w_key)
 
-    def descr_reversed(self, space):
-        raise oefmt(space.w_TypeError,
-                    "argument to reversed() must be a sequence")
-
     def descr_copy(self, space):
         """D.copy() -> a shallow copy of D"""
         return self.copy()
@@ -403,7 +399,6 @@
     __setitem__ = interp2app(W_DictMultiObject.descr_setitem),
     __delitem__ = interp2app(W_DictMultiObject.descr_delitem),
 
-    __reversed__ = interp2app(W_DictMultiObject.descr_reversed),
     copy = interp2app(W_DictMultiObject.descr_copy),
     items = interp2app(W_DictMultiObject.descr_items),
     keys = interp2app(W_DictMultiObject.descr_keys),

From pypy.commits at gmail.com  Wed Nov  2 13:53:29 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 10:53:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: compile() now raises ValueError,
 not TypeError, when the source contains null bytes
Message-ID: <581a2819.8f95c20a.6b64c.5407@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88089:eebb8cd4bece
Date: 2016-11-02 17:52 +0000
http://bitbucket.org/pypy/pypy/changeset/eebb8cd4bece/

Log:	compile() now raises ValueError, not TypeError, when the source
	contains null bytes

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1819,7 +1819,7 @@
 
     if not (flags & consts.PyCF_ACCEPT_NULL_BYTES):
         if '\x00' in source:
-            raise oefmt(space.w_TypeError,
+            raise oefmt(space.w_ValueError,
                         "source code string cannot contain null bytes")
         source = rstring.assert_str0(source)
     return source, flags
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -502,7 +502,7 @@
         assert eval(co) == 3
         co = compile(memoryview(b'1+2'), '?', 'eval')
         assert eval(co) == 3
-        exc = raises(TypeError, compile, chr(0), '?', 'eval')
+        exc = raises(ValueError, compile, chr(0), '?', 'eval')
         assert str(exc.value) == "source code string cannot contain null bytes"
         compile("from __future__ import with_statement", "<test>", "exec")
         raises(SyntaxError, compile, '-', '?', 'eval')
@@ -646,10 +646,10 @@
         assert firstlineno == 2
 
     def test_compile_null_bytes(self):
-        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        raises(ValueError, compile, '\x00', 'mymod', 'exec', 0)
         src = "#abc\x00def\n"
-        raises(TypeError, compile, src, 'mymod', 'exec')
-        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+        raises(ValueError, compile, src, 'mymod', 'exec')
+        raises(ValueError, compile, src, 'mymod', 'exec', 0)
 
     def test_compile_null_bytes_flag(self):
         try:

From pypy.commits at gmail.com  Wed Nov  2 14:21:20 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 11:21:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Move all compile() tests from
 test_builtin.py to test_compile.py
Message-ID: <581a2ea0.a940c20a.45460.630c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88090:dac7bb0e17dd
Date: 2016-11-02 18:05 +0000
http://bitbucket.org/pypy/pypy/changeset/dac7bb0e17dd/

Log:	Move all compile() tests from test_builtin.py to test_compile.py

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -497,73 +497,6 @@
         assert eval("i", None, None) == 4
         assert eval('a', None, dict(a=42)) == 42
 
-    def test_compile(self):
-        co = compile('1+2', '?', 'eval')
-        assert eval(co) == 3
-        co = compile(memoryview(b'1+2'), '?', 'eval')
-        assert eval(co) == 3
-        exc = raises(ValueError, compile, chr(0), '?', 'eval')
-        assert str(exc.value) == "source code string cannot contain null bytes"
-        compile("from __future__ import with_statement", "<test>", "exec")
-        raises(SyntaxError, compile, '-', '?', 'eval')
-        raises(SyntaxError, compile, '"\\xt"', '?', 'eval')
-        raises(ValueError, compile, '1+2', '?', 'maybenot')
-        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
-        raises(TypeError, compile, '1+2', 12, 34)
-
-    def test_compile_error_message(self):
-        import re
-        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
-        exc = raises(SyntaxError, compile,
-            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        assert 'iso-8859-15' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-
-    def test_unicode_compile(self):
-        try:
-            compile(u'-', '?', 'eval')
-        except SyntaxError as e:
-            assert e.lineno == 1
-
-    def test_unicode_encoding_compile(self):
-        code = "# -*- coding: utf-8 -*-\npass\n"
-        compile(code, "tmp", "exec")
-
-    def test_bytes_compile(self):
-        code = b"# -*- coding: utf-8 -*-\npass\n"
-        compile(code, "tmp", "exec")
-        c = compile(b"# coding: latin1\nfoo = 'caf\xe9'\n", "<string>", "exec")
-        ns = {}
-        exec(c, ns)
-        assert ns['foo'] == 'caf&#233;'
-        assert eval(b"# coding: latin1\n'caf\xe9'\n") == 'caf&#233;'
-
-    def test_memoryview_compile(self):
-        m = memoryview(b'2 + 1')
-        co = compile(m, 'baz', 'eval')
-        assert eval(co) == 3
-        assert eval(m) == 3
-        ns = {}
-        exec(memoryview(b'r = 2 + 1'), ns)
-        assert ns['r'] == 3
-
-    def test_recompile_ast(self):
-        import _ast
-        # raise exception when node type doesn't match with compile mode
-        co1 = compile('print(1)', '<string>', 'exec', _ast.PyCF_ONLY_AST)
-        raises(TypeError, compile, co1, '<ast>', 'eval')
-        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
-
     def test_isinstance(self):
         assert isinstance(5, int)
         assert isinstance(5, object)
@@ -637,30 +570,6 @@
         assert hasattr(x, 'foo') is False
         raises(KeyError, "hasattr(x, 'bar')")
 
-    def test_compile_leading_newlines(self):
-        src = """
-def fn(): pass
-"""
-        co = compile(src, 'mymod', 'exec')
-        firstlineno = co.co_firstlineno
-        assert firstlineno == 2
-
-    def test_compile_null_bytes(self):
-        raises(ValueError, compile, '\x00', 'mymod', 'exec', 0)
-        src = "#abc\x00def\n"
-        raises(ValueError, compile, src, 'mymod', 'exec')
-        raises(ValueError, compile, src, 'mymod', 'exec', 0)
-
-    def test_compile_null_bytes_flag(self):
-        try:
-            from _ast import PyCF_ACCEPT_NULL_BYTES
-        except ImportError:
-            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               PyCF_ACCEPT_NULL_BYTES)
-        src = "#abc\x00def\n"
-        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
-
     def test_print_function(self):
         import builtins
         import sys
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,6 +1,98 @@
+# coding: utf-8
 class AppTestCompile:
 
-    def test_compile(self):
+    def test_simple(self):
+        co = compile('1+2', '?', 'eval')
+        assert eval(co) == 3
+        co = compile(memoryview(b'1+2'), '?', 'eval')
+        assert eval(co) == 3
+        exc = raises(ValueError, compile, chr(0), '?', 'eval')
+        assert str(exc.value) == "source code string cannot contain null bytes"
+        compile("from __future__ import with_statement", "<test>", "exec")
+        raises(SyntaxError, compile, '-', '?', 'eval')
+        raises(SyntaxError, compile, '"\\xt"', '?', 'eval')
+        raises(ValueError, compile, '1+2', '?', 'maybenot')
+        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
+        raises(TypeError, compile, '1+2', 12, 34)
+
+    def test_error_message(self):
+        import re
+        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
+        exc = raises(SyntaxError, compile,
+            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        assert 'iso-8859-15' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+
+    def test_unicode(self):
+        try:
+            compile(u'-', '?', 'eval')
+        except SyntaxError as e:
+            assert e.lineno == 1
+
+    def test_unicode_encoding(self):
+        code = "# -*- coding: utf-8 -*-\npass\n"
+        compile(code, "tmp", "exec")
+
+    def test_bytes(self):
+        code = b"# -*- coding: utf-8 -*-\npass\n"
+        compile(code, "tmp", "exec")
+        c = compile(b"# coding: latin1\nfoo = 'caf\xe9'\n", "<string>", "exec")
+        ns = {}
+        exec(c, ns)
+        assert ns['foo'] == 'caf&#233;'
+        assert eval(b"# coding: latin1\n'caf\xe9'\n") == 'caf&#233;'
+
+    def test_memoryview(self):
+        m = memoryview(b'2 + 1')
+        co = compile(m, 'baz', 'eval')
+        assert eval(co) == 3
+        assert eval(m) == 3
+        ns = {}
+        exec(memoryview(b'r = 2 + 1'), ns)
+        assert ns['r'] == 3
+
+    def test_recompile_ast(self):
+        import _ast
+        # raise exception when node type doesn't match with compile mode
+        co1 = compile('print(1)', '<string>', 'exec', _ast.PyCF_ONLY_AST)
+        raises(TypeError, compile, co1, '<ast>', 'eval')
+        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
+        compile(co2, '<ast>', 'eval')
+
+    def test_leading_newlines(self):
+        src = """
+def fn(): pass
+"""
+        co = compile(src, 'mymod', 'exec')
+        firstlineno = co.co_firstlineno
+        assert firstlineno == 2
+
+    def test_null_bytes(self):
+        raises(ValueError, compile, '\x00', 'mymod', 'exec', 0)
+        src = "#abc\x00def\n"
+        raises(ValueError, compile, src, 'mymod', 'exec')
+        raises(ValueError, compile, src, 'mymod', 'exec', 0)
+
+    def test_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
+
+    def test_compile_regression(self):
         """Clone of the part of the original test that was failing."""
         import ast
 

From pypy.commits at gmail.com  Wed Nov  2 14:22:18 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 11:22:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Move all compile() tests from
 test_builtin.py to test_compile.py (backport dac7bb0e17dd)
Message-ID: <581a2eda.e128c20a.60b07.611f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88091:50aae6a02e7f
Date: 2016-11-02 18:20 +0000
http://bitbucket.org/pypy/pypy/changeset/50aae6a02e7f/

Log:	Move all compile() tests from test_builtin.py to test_compile.py
	(backport dac7bb0e17dd)

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -509,59 +509,6 @@
         assert eval("i", None, None) == 4
         assert eval('a', None, dict(a=42)) == 42
 
-    def test_compile(self):
-        co = compile('1+2', '?', 'eval')
-        assert eval(co) == 3
-        co = compile(buffer('1+2'), '?', 'eval')
-        assert eval(co) == 3
-        exc = raises(TypeError, compile, chr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
-        compile("from __future__ import with_statement", "<test>", "exec")
-        raises(SyntaxError, compile, '-', '?', 'eval')
-        raises(ValueError, compile, '"\\xt"', '?', 'eval')
-        raises(ValueError, compile, '1+2', '?', 'maybenot')
-        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
-        raises(TypeError, compile, '1+2', 12, 34)
-
-    def test_compile_error_message(self):
-        import re
-        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
-        exc = raises(SyntaxError, compile,
-            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        assert 'iso-8859-15' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-
-    def test_unicode_compile(self):
-        try:
-            compile(u'-', '?', 'eval')
-        except SyntaxError as e:
-            assert e.lineno == 1
-
-    def test_unicode_encoding_compile(self):
-        code = u"# -*- coding: utf-8 -*-\npass\n"
-        raises(SyntaxError, compile, code, "tmp", "exec")
-
-    def test_recompile_ast(self):
-        import _ast
-        # raise exception when node type doesn't match with compile mode
-        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
-        raises(TypeError, compile, co1, '<ast>', 'eval')
-        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
-
     def test_isinstance(self):
         assert isinstance(5, int)
         assert isinstance(5, object)
@@ -624,34 +571,10 @@
         raises(TypeError, hasattr, x, 42)
         raises(UnicodeError, hasattr, x, u'\u5678')  # cannot encode attr name
 
-    def test_compile_leading_newlines(self):
-        src = """
-def fn(): pass
-"""
-        co = compile(src, 'mymod', 'exec')
-        firstlineno = co.co_firstlineno
-        assert firstlineno == 2
-
-    def test_compile_null_bytes(self):
-        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        src = "#abc\x00def\n"
-        raises(TypeError, compile, src, 'mymod', 'exec')
-        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+    def test_execfile_args(self):
         execfile(self.nullbytes) # works
-
-    def test_execfile_args(self):
         raises(TypeError, execfile, self.nonexistent, {}, ())
 
-    def test_compile_null_bytes_flag(self):
-        try:
-            from _ast import PyCF_ACCEPT_NULL_BYTES
-        except ImportError:
-            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               PyCF_ACCEPT_NULL_BYTES)
-        src = "#abc\x00def\n"
-        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
-
     def test_print_function(self):
         import __builtin__
         import sys
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -0,0 +1,77 @@
+class AppTestCompile:
+    def test_simple(self):
+        co = compile('1+2', '?', 'eval')
+        assert eval(co) == 3
+        co = compile(buffer('1+2'), '?', 'eval')
+        assert eval(co) == 3
+        exc = raises(TypeError, compile, chr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
+        assert str(exc.value) == "expected a readable buffer object"
+        compile("from __future__ import with_statement", "<test>", "exec")
+        raises(SyntaxError, compile, '-', '?', 'eval')
+        raises(ValueError, compile, '"\\xt"', '?', 'eval')
+        raises(ValueError, compile, '1+2', '?', 'maybenot')
+        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
+        raises(TypeError, compile, '1+2', 12, 34)
+
+    def test_error_message(self):
+        import re
+        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
+        exc = raises(SyntaxError, compile,
+            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        assert 'iso-8859-15' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+
+    def test_unicode(self):
+        try:
+            compile(u'-', '?', 'eval')
+        except SyntaxError as e:
+            assert e.lineno == 1
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        raises(SyntaxError, compile, code, "tmp", "exec")
+
+    def test_recompile_ast(self):
+        import _ast
+        # raise exception when node type doesn't match with compile mode
+        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
+        raises(TypeError, compile, co1, '<ast>', 'eval')
+        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
+        compile(co2, '<ast>', 'eval')
+
+    def test_leading_newlines(self):
+        src = """
+def fn(): pass
+"""
+        co = compile(src, 'mymod', 'exec')
+        firstlineno = co.co_firstlineno
+        assert firstlineno == 2
+
+    def test_null_bytes(self):
+        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        src = "#abc\x00def\n"
+        raises(TypeError, compile, src, 'mymod', 'exec')
+        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+
+    def test_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works

From pypy.commits at gmail.com  Wed Nov  2 18:50:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 02 Nov 2016 15:50:47 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <581a6dc7.857bc20a.e9450.be6d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r813:5c870cc3355a
Date: 2016-11-02 23:50 +0100
http://bitbucket.org/pypy/pypy.org/changeset/5c870cc3355a/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66222 of $105000 (63.1%)
+   $66246 of $105000 (63.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $3528 left
+    we have $3549 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Nov  2 21:00:13 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 18:00:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use 3.5 order of evaluation for dict
 literals
Message-ID: <581a8c1d.05371c0a.a767c.ee2a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88092:f205cd1fb012
Date: 2016-11-03 00:59 +0000
http://bitbucket.org/pypy/pypy/changeset/f205cd1fb012/

Log:	Use 3.5 order of evaluation for dict literals

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -662,7 +662,7 @@
     ops.LOAD_DEREF: 1,
     ops.STORE_DEREF: -1,
     ops.DELETE_DEREF: 0,
-    
+
     ops.GET_AWAITABLE: 0,
     ops.SETUP_ASYNC_WITH: 0,
     ops.BEFORE_ASYNC_WITH: 1,
@@ -682,10 +682,10 @@
     ops.JUMP_IF_FALSE_OR_POP: 0,
     ops.POP_JUMP_IF_TRUE: -1,
     ops.POP_JUMP_IF_FALSE: -1,
-    # TODO 
+    # TODO
     ops.JUMP_IF_NOT_DEBUG: 0,
 
-    # TODO 
+    # TODO
     ops.BUILD_LIST_FROM_ARG: 1,
     # TODO
     ops.LOAD_CLASSDEREF: 1,
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1216,8 +1216,8 @@
                     # in some cases, so another error has to be fixed in order for
                     # this to work, otherwise it breaks everything
                     # after fix: remove dirty fixes in pyopcode
+                    key.walkabout(self)
                     d.values[i].walkabout(self)
-                    key.walkabout(self)
                     elements += 1
         if elements or containers == 0:
             self.emit_op_arg(ops.BUILD_MAP, elements)
@@ -1301,8 +1301,8 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
+                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     kw.value.walkabout(self)
-                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     nseen += 1
                 else:
                     # keyword argument
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -792,7 +792,7 @@
             for i in arg:
                 ret.append(i)
             return ret
-        
+
         args = [4,5,6]
         res = call(*args, *args)
         """)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,7 +38,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 96 # bump it by 16
+pypy_incremental_magic = 112 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1327,8 +1327,8 @@
     def BUILD_MAP(self, itemcount, next_instr):
         w_dict = self.space.newdict()
         for i in range(itemcount-1, -1, -1):
-            w_key = self.peekvalue(2 * i)
-            w_value = self.peekvalue(2 * i + 1)
+            w_value = self.peekvalue(2 * i)
+            w_key = self.peekvalue(2 * i + 1)
             self.space.setitem(w_dict, w_key, w_value)
         self.popvalues(2 * itemcount)
         self.pushvalue(w_dict)

From pypy.commits at gmail.com  Wed Nov  2 21:19:34 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 Nov 2016 18:19:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: remove obsolete comment
Message-ID: <581a90a6.c2d21c0a.44b59.f04b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88093:cf3b16633b90
Date: 2016-11-03 01:18 +0000
http://bitbucket.org/pypy/pypy/changeset/cf3b16633b90/

Log:	remove obsolete comment

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1210,12 +1210,6 @@
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
-                    # TODO: key.walkabout has to be called before d.values.walkabout
-                    # that would fix the error "keywords must be strings"
-                    # for some reason the keys and values seem to be in reverse order
-                    # in some cases, so another error has to be fixed in order for
-                    # this to work, otherwise it breaks everything
-                    # after fix: remove dirty fixes in pyopcode
                     key.walkabout(self)
                     d.values[i].walkabout(self)
                     elements += 1

From pypy.commits at gmail.com  Thu Nov  3 02:07:21 2016
From: pypy.commits at gmail.com (mjacob)
Date: Wed, 02 Nov 2016 23:07:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix typo.
Message-ID: <581ad419.838e1c0a.944b4.2e6e@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: 
Changeset: r88094:bacc55bdad61
Date: 2016-11-03 07:06 +0100
http://bitbucket.org/pypy/pypy/changeset/bacc55bdad61/

Log:	Fix typo.

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -16,7 +16,7 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
-We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
 well as maintaining support for previous versions.
 
 XXX

From pypy.commits at gmail.com  Thu Nov  3 04:30:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 01:30:31 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: prevent overflow during test
 which would make the result not equal (python does not wrap around)
Message-ID: <581af5a7.e8edc20a.1018a.44b4@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88095:dbe73c37fc59
Date: 2016-11-03 09:29 +0100
http://bitbucket.org/pypy/pypy/changeset/dbe73c37fc59/

Log:	prevent overflow during test which would make the result not equal
	(python does not wrap around)

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -236,14 +236,14 @@
     vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int)
 
     test_vec_signed_add = \
-        vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED)
+        vec_int_arith(lambda a,b: intmask(intmask(a)+intmask(b)), rffi.SIGNED)
     test_vec_int_add = \
         vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT)
     test_vec_short_add = \
         vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT)
 
-    test_vec_sub_signed = \
-        vec_int_arith(lambda a,b: intmask(a-b), rffi.SIGNED)
+    test_vec_signed_sub = \
+        vec_int_arith(lambda a,b: intmask(intmask(a)-intmask(b)), rffi.SIGNED)
     test_vec_sub_int = \
         vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT)
     test_vec_sub_short = \

From pypy.commits at gmail.com  Thu Nov  3 04:37:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 01:37:51 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: merge default
Message-ID: <581af75f.a940c20a.45460.46aa@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88096:5083544e46fd
Date: 2016-11-03 09:34 +0100
http://bitbucket.org/pypy/pypy/changeset/5083544e46fd/

Log:	merge default

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -16,7 +16,7 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
-We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
 well as maintaining support for previous versions.
 
 XXX
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -81,6 +81,14 @@
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
 
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
 .. branch: zarch-simd-support
 
 s390x implementation for vector operations used in VecOpt
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -509,59 +509,6 @@
         assert eval("i", None, None) == 4
         assert eval('a', None, dict(a=42)) == 42
 
-    def test_compile(self):
-        co = compile('1+2', '?', 'eval')
-        assert eval(co) == 3
-        co = compile(buffer('1+2'), '?', 'eval')
-        assert eval(co) == 3
-        exc = raises(TypeError, compile, chr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
-        compile("from __future__ import with_statement", "<test>", "exec")
-        raises(SyntaxError, compile, '-', '?', 'eval')
-        raises(ValueError, compile, '"\\xt"', '?', 'eval')
-        raises(ValueError, compile, '1+2', '?', 'maybenot')
-        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
-        raises(TypeError, compile, '1+2', 12, 34)
-
-    def test_compile_error_message(self):
-        import re
-        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
-        exc = raises(SyntaxError, compile,
-            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        assert 'iso-8859-15' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-
-    def test_unicode_compile(self):
-        try:
-            compile(u'-', '?', 'eval')
-        except SyntaxError as e:
-            assert e.lineno == 1
-
-    def test_unicode_encoding_compile(self):
-        code = u"# -*- coding: utf-8 -*-\npass\n"
-        raises(SyntaxError, compile, code, "tmp", "exec")
-
-    def test_recompile_ast(self):
-        import _ast
-        # raise exception when node type doesn't match with compile mode
-        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
-        raises(TypeError, compile, co1, '<ast>', 'eval')
-        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
-
     def test_isinstance(self):
         assert isinstance(5, int)
         assert isinstance(5, object)
@@ -624,34 +571,10 @@
         raises(TypeError, hasattr, x, 42)
         raises(UnicodeError, hasattr, x, u'\u5678')  # cannot encode attr name
 
-    def test_compile_leading_newlines(self):
-        src = """
-def fn(): pass
-"""
-        co = compile(src, 'mymod', 'exec')
-        firstlineno = co.co_firstlineno
-        assert firstlineno == 2
-
-    def test_compile_null_bytes(self):
-        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        src = "#abc\x00def\n"
-        raises(TypeError, compile, src, 'mymod', 'exec')
-        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+    def test_execfile_args(self):
         execfile(self.nullbytes) # works
-
-    def test_execfile_args(self):
         raises(TypeError, execfile, self.nonexistent, {}, ())
 
-    def test_compile_null_bytes_flag(self):
-        try:
-            from _ast import PyCF_ACCEPT_NULL_BYTES
-        except ImportError:
-            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               PyCF_ACCEPT_NULL_BYTES)
-        src = "#abc\x00def\n"
-        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
-
     def test_print_function(self):
         import __builtin__
         import sys
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -0,0 +1,77 @@
+class AppTestCompile:
+    def test_simple(self):
+        co = compile('1+2', '?', 'eval')
+        assert eval(co) == 3
+        co = compile(buffer('1+2'), '?', 'eval')
+        assert eval(co) == 3
+        exc = raises(TypeError, compile, chr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
+        assert str(exc.value) == "expected a readable buffer object"
+        compile("from __future__ import with_statement", "<test>", "exec")
+        raises(SyntaxError, compile, '-', '?', 'eval')
+        raises(ValueError, compile, '"\\xt"', '?', 'eval')
+        raises(ValueError, compile, '1+2', '?', 'maybenot')
+        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
+        raises(TypeError, compile, '1+2', 12, 34)
+
+    def test_error_message(self):
+        import re
+        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
+        exc = raises(SyntaxError, compile,
+            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        assert 'iso-8859-15' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+
+    def test_unicode(self):
+        try:
+            compile(u'-', '?', 'eval')
+        except SyntaxError as e:
+            assert e.lineno == 1
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        raises(SyntaxError, compile, code, "tmp", "exec")
+
+    def test_recompile_ast(self):
+        import _ast
+        # raise exception when node type doesn't match with compile mode
+        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
+        raises(TypeError, compile, co1, '<ast>', 'eval')
+        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
+        compile(co2, '<ast>', 'eval')
+
+    def test_leading_newlines(self):
+        src = """
+def fn(): pass
+"""
+        co = compile(src, 'mymod', 'exec')
+        firstlineno = co.co_firstlineno
+        assert firstlineno == 2
+
+    def test_null_bytes(self):
+        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        src = "#abc\x00def\n"
+        raises(TypeError, compile, src, 'mymod', 'exec')
+        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+
+    def test_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -479,6 +479,14 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recv_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recv_into(buffer, [nbytes[, flags]]) -> nbytes_read
+
+        A version of recv() that stores its data into a buffer rather than creating
+        a new string.  Receive up to buffersize bytes from the socket.  If buffersize
+        is not specified (or 0), receive up to the size available in the given buffer.
+
+        See recv() for documentation about the flags.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0 or nbytes > lgt:
@@ -490,6 +498,10 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recvfrom_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)
+
+        Like recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0:
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1362,8 +1362,14 @@
 def _time_to_timeval(t, l_timeval):
     import math
     fracpart, intpart = math.modf(t)
-    rffi.setintfield(l_timeval, 'c_tv_sec', int(intpart))
-    rffi.setintfield(l_timeval, 'c_tv_usec', int(fracpart * 1e6))
+    intpart = int(intpart)
+    fracpart = int(fracpart * 1e6)
+    if fracpart < 0:
+        intpart -= 1
+        fracpart += 1000000
+    assert 0 <= fracpart < 1000000
+    rffi.setintfield(l_timeval, 'c_tv_sec', intpart)
+    rffi.setintfield(l_timeval, 'c_tv_usec', fracpart)
 
 if not _WIN32:
     TMSP = lltype.Ptr(TMS)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -59,6 +59,17 @@
         compile(f, (str, float))(str(fname), t1)
         assert t1 == os.stat(str(fname)).st_mtime
 
+    def test_utime_negative_fraction(self):
+        def f(fname, t1):
+            os.utime(fname, (t1, t1))
+
+        fname = udir.join('test_utime_negative_fraction.txt')
+        fname.ensure()
+        t1 = -123.75
+        compile(f, (str, float))(str(fname), t1)
+        got = os.stat(str(fname)).st_mtime
+        assert got == -123 or got == -123.75
+
     @win_only
     def test__getfullpathname(self):
         posix = __import__(os.name)

From pypy.commits at gmail.com  Thu Nov  3 04:37:53 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 01:37:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge ppc-vsx-support and
 s390x-z-simd-support
Message-ID: <581af761.876ec20a.12fc1.342a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88097:9f66930d64e8
Date: 2016-11-03 09:36 +0100
http://bitbucket.org/pypy/pypy/changeset/9f66930d64e8/

Log:	merge ppc-vsx-support and s390x-z-simd-support

diff too long, truncating to 2000 out of 7995 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -81,6 +81,7 @@
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
 
+
 .. branch: fix-struct-unpack-Q
 
 Improve compatibility with CPython in the ``struct`` module. In particular,
@@ -88,12 +89,10 @@
 while previously it always returned a ``long`` for certains format codes such
 as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
 
-.. branch: newinitwarn
+.. branch: zarch-simd-support
 
-Issue warnings for stricter handling of ``__new/init__`` args (that
-become TypeErrors in python 3)
+s390x implementation for vector operations used in VecOpt
 
-.. branch: openssl-1.1
+.. branch: ppc-vsx-support
 
-PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
-installed.  Thanks tumbleweed!
+PowerPC implementation for vector operations used in VecOpt
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -16,7 +16,7 @@
 from rpython.jit.backend.detect_cpu import getcpuclass
 
 CPU = getcpuclass()
-if not CPU.vector_extension:
+if not CPU.vector_ext:
     py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
 def get_profiler():
@@ -29,7 +29,7 @@
     interp = None
 
     def setup_method(self, method):
-        if not self.CPUClass.vector_extension:
+        if not self.CPUClass.vector_ext:
             py.test.skip("needs vector extension to run (for now)")
 
     def assert_float_equal(self, f1, f2, delta=0.0001):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -3,23 +3,35 @@
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 from rpython.rlib.rawstorage import misaligned_is_fine
 
+def no_vector_backend():
+    import platform
+    if platform.machine().startswith('x86'):
+        from rpython.jit.backend.x86.detect_feature import detect_sse4_2
+        return not detect_sse4_2()
+    if platform.machine().startswith('ppc'):
+        from rpython.jit.backend.ppc.detect_feature import detect_vsx
+        return not detect_vsx()
+    if platform.machine() == "s390x":
+        from rpython.jit.backend.zarch.detect_feature import detect_simd_z
+        return not detect_simd_z()
+    return True
 
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
-             ('+','float','int',   9*7843,   7843, 4.0,5.0),
-             ('+','int','float',   8*2571,   2571, 9.0,-1.0),
-             ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
-             ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
-             ('-','float','float', -2*5523,  5523, 1.0,3.0),
-             ('*','float','float', 3*2999,   2999, 1.0,3.0),
-             ('/','float','float', 3*7632,   7632, 3.0,1.0),
-             ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
-             ('&','int','int',     0,        1500, 1,0),
-             ('&','int','int',     1500,     1500, 1,1),
-             ('|','int','int',     1500,     1500, 0,1),
-             ('|','int','int',     0,        1500, 0,0),
-            ]
+                  ('+','float','int',   9*7843,   7843, 4.0,5.0),
+                  ('+','int','float',   8*2571,   2571, 9.0,-1.0),
+                  ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
+                  ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
+                  ('-','float','float', -2*5523,  5523, 1.0,3.0),
+                  ('*','float','float', 3*2999,   2999, 1.0,3.0),
+                  ('/','float','float', 3*7632,   7632, 3.0,1.0),
+                  ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
+                  ('&','int','int',     0,        1500, 1,0),
+                  ('&','int','int',     1500,     1500, 1,1),
+                  ('|','int','int',     1500,     1500, 0,1),
+                  ('|','int','int',     0,        1500, 0,0),
+                 ]
     type_permuated = []
     types = { 'int': ['int32','int64','int8','int16'],
               'float': ['float32', 'float64']
@@ -36,6 +48,7 @@
                 type_permuated.append(t)
 
     @py.test.mark.parametrize("op,adtype,bdtype,result,count,a,b", type_permuated)
+    @py.test.mark.skipif('no_vector_backend()')
     def test_vector_call2(self, op, adtype, bdtype, result, count, a, b):
         source = """
         def main():
@@ -51,6 +64,13 @@
         log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        if adtype in ('int64','float64') and bdtype in ('int64','float64'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
 
     arith_comb = [
@@ -58,7 +78,7 @@
         ('sum','float', 2581, 2581, 1),
         ('prod','float', 1, 3178, 1),
         ('prod','int', 1, 3178, 1),
-        ('any','int', 1, 1239, 1),
+        ('any','int', 1, 2239, 1),
         ('any','int', 0, 4912, 0),
         ('all','int', 0, 3420, 0),
         ('all','int', 1, 6757, 1),
@@ -76,6 +96,7 @@
             type_permuated.append(t)
 
     @py.test.mark.parametrize("op,dtype,result,count,a", type_permuated)
+    @py.test.mark.skipif('no_vector_backend()')
     def test_reduce_generic(self,op,dtype,result,count,a):
         source = """
         def main():
@@ -84,10 +105,19 @@
             return a.{method}()
         """.format(method=op, dtype=dtype, count=count, a=a)
         exec py.code.Source(source).compile()
+        log = self.run(main, [], vec=0)
         vlog = self.run(main, [], vec=1)
-        log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        if not log.jit_summary:
+            return
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        if dtype in ('int64','float64') and (dtype != 'int64' and op != 'prod'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
     def test_reduce_logical_xor(self):
         def main():
@@ -158,7 +188,7 @@
         assert log.result is True
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
-        assert loop.match("""
+        loop.match("""
             f31 = raw_load_f(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
@@ -168,7 +198,20 @@
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
             jump(..., descr=...)
-        """)
+            """)
+        # vector version
+        #assert loop.match("""
+        #    guard_not_invalidated(descr=...)
+        #    i38 = int_add(i25, 2)
+        #    i39 = int_ge(i38, i33)
+        #    guard_false(i39, descr=...)
+        #    v42 = vec_load_f(i9, i32, 1, 0, descr=<ArrayF 8>)
+        #    v43 = vec_float_ne(v42, v36)
+        #    f46 = vec_unpack_f(v42, 0, 1)
+        #    vec_guard_true(v43, descr=...)
+        #    i48 = int_add(i32, 16)
+        #    i50 = int_add(i25, 2)
+        #    jump(..., descr=...)""")
 
     def test_array_getitem_basic(self):
         def main():
diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -13,11 +13,6 @@
   (e.g. those in the NumPyPy module).
 * --jit vec_all=1: turns on the vectorization for any jit driver. See parameters for
   the filtering heuristics of traces.
-* --jit vec_ratio=2: A number from 0 to 10 that represents a real number (vec_ratio / 10).
-  This filters traces if vec_all is enabled. N is the trace count then the number of
-  vector transformable operations (add_int -> vec_add_int) M, the following must hold:
-  M / N >= (vec_ratio / 10)
-* --jit vec_length=60: The maximum number of trace instructions the vectorizer filters for.
 
 Features
 --------
diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -13,7 +13,6 @@
 MODEL_X86         = 'x86'
 MODEL_X86_NO_SSE2 = 'x86-without-sse2'
 MODEL_X86_64      = 'x86-64'
-MODEL_X86_64_SSE4 = 'x86-64-sse4'
 MODEL_ARM         = 'arm'
 MODEL_PPC_64      = 'ppc-64'
 MODEL_S390_64     = 's390x'
@@ -81,9 +80,6 @@
         from rpython.jit.backend.x86 import detect_feature as feature
         if sys.maxint == 2**63-1:
             result = MODEL_X86_64
-            # has sse 2 at least
-            if feature.detect_sse4_1():
-                result = MODEL_X86_64_SSE4
         else:
             assert sys.maxint == 2**31-1
             if feature.detect_sse2():
@@ -120,8 +116,6 @@
         return "rpython.jit.backend.x86.runner", "CPU386_NO_SSE2"
     elif backend_name == MODEL_X86_64:
         return "rpython.jit.backend.x86.runner", "CPU_X86_64"
-    elif backend_name == MODEL_X86_64_SSE4:
-        return "rpython.jit.backend.x86.runner", "CPU_X86_64_SSE4"
     elif backend_name == MODEL_ARM:
         return "rpython.jit.backend.arm.runner", "CPU_ARM"
     elif backend_name == MODEL_PPC_64:
@@ -145,7 +139,6 @@
         MODEL_X86: ['floats', 'singlefloats', 'longlong'],
         MODEL_X86_NO_SSE2: ['longlong'],
         MODEL_X86_64: ['floats', 'singlefloats'],
-        MODEL_X86_64_SSE4: ['floats', 'singlefloats'],
         MODEL_ARM: ['floats', 'singlefloats', 'longlong'],
         MODEL_PPC_64: ['floats'],
         MODEL_S390_64: ['floats'],
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend import model
 from rpython.jit.backend.llgraph import support
 from rpython.jit.backend.llsupport import symbolic
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.jit.metainterp.history import Const, getkind
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID
@@ -328,10 +329,11 @@
     supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
-    vector_extension = True
-    vector_register_size = 16 # in bytes
-    vector_horizontal_operations = True
-    vector_pack_slots = True
+    vector_ext = VectorExt()
+    vector_ext.enable(16, accum=True)
+    vector_ext.setup_once = lambda asm: asm
+    load_supported_factors = (1,2,4,8)
+    assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
         model.AbstractCPU.__init__(self)
@@ -877,6 +879,9 @@
     def bh_vec_int_xor(self, vx, vy, count):
         return [int(x) ^ int(y) for x,y in zip(vx,vy)]
 
+    def bh_vec_float_xor(self, vx, vy, count):
+        return [0.0 for x,y in zip(vx,vy)] # just used for clearing the vector register
+
     def bh_vec_cast_float_to_singlefloat(self, vx, count):
         from rpython.rlib.rarithmetic import r_singlefloat
         return [longlong.singlefloat2int(r_singlefloat(longlong.getrealfloat(v)))
@@ -928,50 +933,32 @@
     def bh_vec_int_signext(self, vx, ext, count):
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
-    def build_getarrayitem(func):
-        def method(self, struct, offset, descr, _count):
+    def build_load(func):
+        def method(self, struct, offset, scale, disp, descr, _count):
             values = []
-            count = self.vector_register_size // descr.get_item_size_in_bytes()
+            count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
+            adr = struct + (offset * scale + disp)
+            a = support.cast_arg(lltype.Ptr(descr.A), adr)
+            array = a._obj
             for i in range(count):
-                val = func(self, struct, offset + i, descr)
+                val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
         return method
 
-    bh_vec_getarrayitem_gc_i = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_gc_f = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_raw_i = build_getarrayitem(bh_getarrayitem_raw)
-    bh_vec_getarrayitem_raw_f = build_getarrayitem(bh_getarrayitem_raw)
-    del build_getarrayitem
+    bh_vec_load_i = build_load(bh_getarrayitem_raw)
+    bh_vec_load_f = build_load(bh_getarrayitem_raw)
+    del build_load
 
-    def _bh_vec_raw_load(self, struct, offset, descr, _count):
-        values = []
+    def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
         stride = descr.get_item_size_in_bytes()
-        count = self.vector_register_size // descr.get_item_size_in_bytes()
-        assert _count == count
-        assert count > 0
-        for i in range(count):
-            val = self.bh_raw_load(struct, offset + i*stride, descr)
-            values.append(val)
-        return values
-
-    bh_vec_raw_load_i = _bh_vec_raw_load
-    bh_vec_raw_load_f = _bh_vec_raw_load
-
-    def bh_vec_raw_store(self, struct, offset, newvalues, descr, count):
-        stride = descr.get_item_size_in_bytes()
+        adr = struct + (offset * scale + disp)
+        a = support.cast_arg(lltype.Ptr(descr.A), adr)
+        array = a._obj
         for i,n in enumerate(newvalues):
-            self.bh_raw_store(struct, offset + i*stride, n, descr)
-
-    def bh_vec_setarrayitem_raw(self, struct, offset, newvalues, descr, count):
-        for i,n in enumerate(newvalues):
-            self.bh_setarrayitem_raw(struct, offset + i, n, descr)
-
-    def bh_vec_setarrayitem_gc(self, struct, offset, newvalues, descr, count):
-        for i,n in enumerate(newvalues):
-            self.bh_setarrayitem_gc(struct, offset + i, n, descr)
+            array.setitem(i, support.cast_arg(descr.A.OF, n))
 
     def store_fail_descr(self, deadframe, descr):
         pass # I *think*
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -681,7 +681,6 @@
     sign = arraydescr.is_item_signed()
     return size, ofs, sign
 
-
 def unpack_fielddescr(fielddescr):
     assert isinstance(fielddescr, FieldDescr)
     ofs = fielddescr.offset
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -35,10 +35,7 @@
     # can an ISA instruction handle a factor to the offset?
     load_supported_factors = (1,)
 
-    vector_extension = False
-    vector_register_size = 0 # in bytes
-    vector_horizontal_operations = False
-    vector_pack_slots = False
+    vector_ext = None
 
     def __init__(self, rtyper, stats, opts, translate_support_code=False,
                  gcdescr=None):
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -349,6 +349,8 @@
         assert len(self.temp_boxes) == 0
         if self.longevity:
             for v in self.reg_bindings:
+                if v not in self.longevity:
+                    llop.debug_print(lltype.Void, "variable %s not in longevity\n" % v.repr({}))
                 assert self.longevity[v][1] > self.position
 
     def try_allocate_reg(self, v, selected_reg=None, need_lower_byte=False):
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -1,5 +1,5 @@
 from rpython.rlib import rgc
-from rpython.rlib.objectmodel import we_are_translated, r_dict
+from rpython.rlib.objectmodel import we_are_translated, r_dict, always_inline
 from rpython.rlib.rarithmetic import ovfcheck, highest_bit
 from rpython.rtyper.lltypesystem import llmemory, lltype, rstr
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
@@ -15,6 +15,7 @@
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.rtyper.lltypesystem.lloperation import llop
 
 FLAG_ARRAY = 0
 FLAG_STR = 1
@@ -157,32 +158,12 @@
         index_box = op.getarg(1)
         self.emit_gc_load_or_indexed(op, ptr_box, index_box, itemsize, itemsize, ofs, sign)
 
-    def handle_rawload(self, op):
-        itemsize, ofs, sign = unpack_arraydescr(op.getdescr())
-        ptr_box = op.getarg(0)
-        index_box = op.getarg(1)
-        self.emit_gc_load_or_indexed(op, ptr_box, index_box, itemsize, 1, ofs, sign)
-
     def _emit_mul_if_factor_offset_not_supported(self, index_box,
                                                  factor, offset):
-        # Returns (factor, offset, index_box) where index_box is either
-        # a non-constant BoxInt or None.
-        if isinstance(index_box, ConstInt):
-            return 1, index_box.value * factor + offset, None
-        else:
-            if factor != 1 and factor not in self.cpu.load_supported_factors:
-                # the factor is supported by the cpu
-                # x & (x - 1) == 0 is a quick test for power of 2
-                assert factor > 0
-                if (factor & (factor - 1)) == 0:
-                    index_box = ResOperation(rop.INT_LSHIFT,
-                            [index_box, ConstInt(highest_bit(factor))])
-                else:
-                    index_box = ResOperation(rop.INT_MUL,
-                            [index_box, ConstInt(factor)])
-                self.emit_op(index_box)
-                factor = 1
-            return factor, offset, index_box
+        factor, offset, new_index_box, emit = cpu_simplify_scale(self.cpu, index_box, factor, offset)
+        if emit:
+            self.emit_op(new_index_box)
+        return factor, offset, new_index_box
 
     def emit_gc_load_or_indexed(self, op, ptr_box, index_box, itemsize,
                                 factor, offset, sign, type='i'):
@@ -214,14 +195,6 @@
         NOT_SIGNED = 0
         CINT_ZERO = ConstInt(0)
         opnum = op.getopnum()
-        #if opnum == rop.CALL_MALLOC_NURSERY_VARSIZE:
-        #    v_length = op.getarg(2)
-        #    scale = op.getarg(1).getint()
-        #    if scale not in self.cpu.load_supported_factors:
-        #        scale, offset, v_length = \
-        #                self._emit_mul_if_factor_offset_not_supported(v_length, scale, 0)
-        #        op.setarg(1, ConstInt(scale))
-        #        op.setarg(2, v_length)
         if rop.is_getarrayitem(opnum) or \
            opnum in (rop.GETARRAYITEM_RAW_I,
                      rop.GETARRAYITEM_RAW_F):
@@ -330,7 +303,11 @@
         self._changed_op = None
         for i in range(len(operations)):
             op = operations[i]
-            assert op.get_forwarded() is None
+            if op.get_forwarded():
+                msg = '[rewrite] operations at %d has forwarded info %s\n' % (i, op.repr({}))
+                if we_are_translated():
+                    llop.debug_print(lltype.Void, msg)
+                raise NotImplementedError(msg)
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             if op is self._changed_op:
@@ -833,10 +810,6 @@
              arraydescr.lendescr.offset != gc_descr.standard_array_length_ofs)):
             return False
         self.emitting_an_operation_that_can_collect()
-        #scale = itemsize
-        #if scale not in self.cpu.load_supported_factors:
-        #    scale, offset, v_length = \
-        #            self._emit_mul_if_factor_offset_not_supported(v_length, scale, 0)
         op = ResOperation(rop.CALL_MALLOC_NURSERY_VARSIZE,
                           [ConstInt(kind), ConstInt(itemsize), v_length],
                           descr=arraydescr)
@@ -1015,3 +988,24 @@
             self._newops.append(load_op)
             self.gcrefs_recently_loaded[index] = load_op
         return load_op
+
+ at always_inline
+def cpu_simplify_scale(cpu, index_box, factor, offset):
+    # Returns (factor, offset, index_box, [ops]) where index_box is either
+    # a non-constant BoxInt or None.
+    if isinstance(index_box, ConstInt):
+        return 1, index_box.value * factor + offset, None, False
+    else:
+        if factor != 1 and factor not in cpu.load_supported_factors:
+            # the factor is supported by the cpu
+            # x & (x - 1) == 0 is a quick test for power of 2
+            assert factor > 0
+            if (factor & (factor - 1)) == 0:
+                index_box = ResOperation(rop.INT_LSHIFT,
+                        [index_box, ConstInt(highest_bit(factor))])
+            else:
+                index_box = ResOperation(rop.INT_MUL,
+                        [index_box, ConstInt(factor)])
+            return 1, offset, index_box, True
+        return factor, offset, index_box, False
+
diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -0,0 +1,294 @@
+from rpython.jit.backend.llsupport.rewrite import cpu_simplify_scale
+from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
+        unpack_fielddescr, unpack_interiorfielddescr, ArrayDescr)
+from rpython.rlib.objectmodel import specialize, always_inline
+from rpython.jit.metainterp.history import (VECTOR, FLOAT, INT, ConstInt)
+from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.optimizeopt.schedule import (forwarded_vecinfo,
+        failnbail_transformation)
+from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
+
+class TypeRestrict(object):
+    ANY_TYPE = '\x00'
+    ANY_SIZE = -1
+    ANY_SIGN = -1
+    ANY_COUNT = -1
+    SIGNED = 1
+    UNSIGNED = 0
+
+    def __init__(self,
+                 type=ANY_TYPE,
+                 bytesize=ANY_SIZE,
+                 count=ANY_SIGN,
+                 sign=ANY_COUNT):
+        self.type = type
+        self.bytesize = bytesize
+        self.sign = sign
+        self.count = count
+
+    @always_inline
+    def any_size(self):
+        return self.bytesize == TypeRestrict.ANY_SIZE
+
+    @always_inline
+    def any_count(self):
+        return self.count == TypeRestrict.ANY_COUNT
+
+    def check(self, value):
+        vecinfo = forwarded_vecinfo(value)
+        assert vecinfo.datatype != '\x00'
+        if self.type != TypeRestrict.ANY_TYPE:
+            if self.type != vecinfo.datatype:
+                msg = "type mismatch %s != %s" % \
+                        (self.type, vecinfo.datatype)
+                failnbail_transformation(msg)
+        assert vecinfo.bytesize > 0
+        if not self.any_size():
+            if self.bytesize != vecinfo.bytesize:
+                msg = "bytesize mismatch %s != %s" % \
+                        (self.bytesize, vecinfo.bytesize)
+                failnbail_transformation(msg)
+        assert vecinfo.count > 0
+        if self.count != TypeRestrict.ANY_COUNT:
+            if vecinfo.count < self.count:
+                msg = "count mismatch %s < %s" % \
+                        (self.count, vecinfo.count)
+                failnbail_transformation(msg)
+        if self.sign != TypeRestrict.ANY_SIGN:
+            if bool(self.sign) == vecinfo.sign:
+                msg = "sign mismatch %s < %s" % \
+                        (self.sign, vecinfo.sign)
+                failnbail_transformation(msg)
+
+    def max_input_count(self, count):
+        """ How many """
+        if self.count != TypeRestrict.ANY_COUNT:
+            return self.count
+        return count
+
+class OpRestrict(object):
+    def __init__(self, argument_restris):
+        self.argument_restrictions = argument_restris
+
+    def check_operation(self, state, pack, op):
+        return None
+
+    def crop_vector(self, op, newsize, size):
+        return newsize, size
+
+    def must_crop_vector(self, op, index):
+        restrict = self.argument_restrictions[index]
+        vecinfo = forwarded_vecinfo(op.getarg(index))
+        size = vecinfo.bytesize
+        newsize = self.crop_to_size(op, index)
+        return not restrict.any_size() and newsize != size
+
+    @always_inline
+    def crop_to_size(self, op, index):
+        restrict = self.argument_restrictions[index]
+        return restrict.bytesize
+
+    def opcount_filling_vector_register(self, op, vec_reg_size):
+        """ How many operations of that kind can one execute
+            with a machine instruction of register size X?
+        """
+        if op.is_typecast():
+            if op.casts_down():
+                size = op.cast_input_bytesize(vec_reg_size)
+                return size // op.cast_from_bytesize()
+            else:
+                return vec_reg_size // op.cast_to_bytesize()
+        vecinfo = forwarded_vecinfo(op)
+        return  vec_reg_size // vecinfo.bytesize
+
+class GuardRestrict(OpRestrict):
+    def opcount_filling_vector_register(self, op, vec_reg_size):
+        arg = op.getarg(0)
+        vecinfo = forwarded_vecinfo(arg)
+        return vec_reg_size // vecinfo.bytesize
+
+class LoadRestrict(OpRestrict):
+    def check_operation(self, state, pack, op):
+        opnum = op.getopnum()
+        descr = op.getdescr()
+        if not we_are_translated() and not isinstance(descr, ArrayDescr):
+            itemsize = descr.get_item_size_in_bytes()
+            ofs = 0
+        else:
+            itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
+        args = [op.getarg(0), op.getarg(1), ConstInt(1), ConstInt(ofs)]
+        if rop.is_getarrayitem(opnum) or \
+             opnum in (rop.GETARRAYITEM_RAW_I, rop.GETARRAYITEM_RAW_F):
+            index_box = op.getarg(1)
+            scale, offset, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs)
+            args[2] = ConstInt(scale)
+            args[3] = ConstInt(offset)
+            if emit:
+                state.oplist.append(changed)
+                args[1] = changed
+
+        return args
+
+
+    def opcount_filling_vector_register(self, op, vec_reg_size):
+        assert rop.is_primitive_load(op.opnum)
+        descr = op.getdescr()
+        return vec_reg_size // descr.get_item_size_in_bytes()
+
+class StoreRestrict(OpRestrict):
+    def __init__(self, argument_restris):
+        self.argument_restrictions = argument_restris
+
+    def check_operation(self, state, pack, op):
+        opnum = op.getopnum()
+        descr = op.getdescr()
+        if not we_are_translated() and not isinstance(descr, ArrayDescr):
+            itemsize = descr.get_item_size_in_bytes()
+            ofs = 0
+        else:
+            itemsize, ofs, _ = unpack_arraydescr(op.getdescr())
+        args = [op.getarg(0), op.getarg(1), op.getarg(2), ConstInt(1), ConstInt(ofs)]
+        if opnum in (rop.SETARRAYITEM_GC, rop.SETARRAYITEM_RAW):
+            index_box = op.getarg(1)
+            scale, offset, changed, emit = cpu_simplify_scale(state.cpu, index_box, itemsize, ofs)
+            args[3] = ConstInt(scale)
+            args[4] = ConstInt(offset)
+            if emit:
+                state.oplist.append(changed)
+                args[1] = changed
+        return args
+
+    def must_crop_vector(self, op, index):
+        vecinfo = forwarded_vecinfo(op.getarg(index))
+        bytesize = vecinfo.bytesize
+        return self.crop_to_size(op, index) != bytesize
+
+    @always_inline
+    def crop_to_size(self, op, index):
+        # there is only one parameter that needs to be transformed!
+        descr = op.getdescr()
+        return descr.get_item_size_in_bytes()
+
+    def opcount_filling_vector_register(self, op, vec_reg_size):
+        assert rop.is_primitive_store(op.opnum)
+        descr = op.getdescr()
+        return vec_reg_size // descr.get_item_size_in_bytes()
+
+class OpMatchSizeTypeFirst(OpRestrict):
+    def check_operation(self, state, pack, op):
+        i = 0
+        infos = [forwarded_vecinfo(o) for o in op.getarglist()]
+        arg0 = op.getarg(i)
+        while arg0.is_constant() and i < op.numargs():
+            i += 1
+            arg0 = op.getarg(i)
+        vecinfo = forwarded_vecinfo(arg0)
+        bytesize = vecinfo.bytesize
+        datatype = vecinfo.datatype
+
+        for arg in op.getarglist():
+            if arg.is_constant():
+                continue
+            curvecinfo = forwarded_vecinfo(arg)
+            if curvecinfo.bytesize != bytesize:
+                debug_print("op match size first type failed")
+                raise NotAVectorizeableLoop
+            if curvecinfo.datatype != datatype:
+                debug_print("op match size first type failed (datatype)")
+                raise NotAVectorizeableLoop
+        return None
+
+TR_ANY = TypeRestrict()
+TR_ANY_FLOAT = TypeRestrict(FLOAT)
+TR_ANY_INTEGER = TypeRestrict(INT)
+TR_FLOAT_2 = TypeRestrict(FLOAT, 4, 2)
+TR_DOUBLE_2 = TypeRestrict(FLOAT, 8, 2)
+TR_INT32_2 = TypeRestrict(INT, 4, 2)
+
+OR_MSTF_I = OpMatchSizeTypeFirst([TR_ANY_INTEGER, TR_ANY_INTEGER])
+OR_MSTF_F = OpMatchSizeTypeFirst([TR_ANY_FLOAT, TR_ANY_FLOAT])
+STORE_RESTRICT = StoreRestrict([None, None, TR_ANY])
+LOAD_RESTRICT = LoadRestrict([])
+GUARD_RESTRICT = GuardRestrict([TR_ANY_INTEGER])
+
+
+class VectorExt(object):
+
+    should_align_unroll = True
+
+    def __init__(self):
+        self._enabled = False
+        self.register_size = 0 # in bytes
+        self.accum = False
+        self._setup = False
+
+    def is_setup(self):
+        return self._setup
+
+    def setup_once(self):
+        raise NotImplementedError
+
+    def enable(self, vec_size, accum=False):
+        self._enabled = vec_size != 0
+        self.register_size = vec_size
+        self.accum = accum
+
+    def is_enabled(self):
+        return self._enabled
+
+    def vec_size(self):
+        return self.register_size
+
+    def supports_accumulation(self):
+        return self.accum
+
+    # note that the following definition is x86 arch specific
+    TR_MAPPING = {
+        rop.VEC_INT_ADD:            OR_MSTF_I,
+        rop.VEC_INT_SUB:            OR_MSTF_I,
+        rop.VEC_INT_MUL:            OR_MSTF_I,
+        rop.VEC_INT_AND:            OR_MSTF_I,
+        rop.VEC_INT_OR:             OR_MSTF_I,
+        rop.VEC_INT_XOR:            OR_MSTF_I,
+        rop.VEC_INT_EQ:             OR_MSTF_I,
+        rop.VEC_INT_NE:             OR_MSTF_I,
+
+        rop.VEC_FLOAT_ADD:          OR_MSTF_F,
+        rop.VEC_FLOAT_SUB:          OR_MSTF_F,
+        rop.VEC_FLOAT_MUL:          OR_MSTF_F,
+        rop.VEC_FLOAT_TRUEDIV:      OR_MSTF_F,
+        rop.VEC_FLOAT_ABS:          OpRestrict([TR_ANY_FLOAT]),
+        rop.VEC_FLOAT_NEG:          OpRestrict([TR_ANY_FLOAT]),
+
+        rop.VEC_STORE:              STORE_RESTRICT,
+
+        rop.VEC_LOAD_I:             LOAD_RESTRICT,
+        rop.VEC_LOAD_F:             LOAD_RESTRICT,
+
+        rop.VEC_GUARD_TRUE:             GUARD_RESTRICT,
+        rop.VEC_GUARD_FALSE:            GUARD_RESTRICT,
+
+        ## irregular
+        rop.VEC_INT_SIGNEXT:        OpRestrict([TR_ANY_INTEGER]),
+
+        rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT:  OpRestrict([TR_DOUBLE_2]),
+        # weird but the trace will store single floats in int boxes
+        rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT:  OpRestrict([TR_INT32_2]),
+        rop.VEC_CAST_FLOAT_TO_INT:          OpRestrict([TR_DOUBLE_2]),
+        rop.VEC_CAST_INT_TO_FLOAT:          OpRestrict([TR_INT32_2]),
+
+        rop.VEC_FLOAT_EQ:           OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]),
+        rop.VEC_FLOAT_NE:           OpRestrict([TR_ANY_FLOAT,TR_ANY_FLOAT]),
+        rop.VEC_INT_IS_TRUE:        OpRestrict([TR_ANY_INTEGER,TR_ANY_INTEGER]),
+    }
+
+    def get_operation_restriction(self, op):
+        res = self.TR_MAPPING.get(op.vector, None)
+        if not res:
+            failnbail_transformation("could not get OpRestrict for " + str(op))
+        return res
+
diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -1,4 +1,5 @@
 import os
+
 from rpython.jit.backend.ppc.ppc_form import PPCForm as Form
 from rpython.jit.backend.ppc.locations import RegisterLocation
 from rpython.jit.backend.ppc.ppc_field import ppc_fields
@@ -60,6 +61,17 @@
 XFL = Form("FM", "frB", "XO1", "Rc")
 XFX = Form("CRM", "rS", "XO1")
 XLL = Form("LL", "XO1")
+XX1 = Form("fvrT", "rA", "rB", "XO1")
+XX2 = Form("fvrT", "fvrB", "XO6")
+XX3 = Form("fvrT", "fvrA", "fvrB", "XO9")
+XX3_2 = Form("fvrT", "fvrA", "fvrB", "OE", "XO11")
+XX3_splat = Form("fvrT", "fvrA", "fvrB", "DM", "XO13", "OE")
+XV = Form("ivrT", "rA", "rB", "XO1")
+VX = Form("ivrT", "ivrA", "ivrB", "XO8")
+VC = Form("ivrT", "ivrA", "ivrB", "XO12", "OE")
+VXI = Form("ivrT", "SIM", "XO8")
+VA = Form("ivrT", "ivrA", "ivrB", "ivrC", "XO10")
+
 
 MI = Form("rA", "rS", "SH", "MB", "ME", "Rc")
 MB = Form("rA", "rS", "rB", "MB", "ME", "Rc")
@@ -568,6 +580,145 @@
     xor = XS(31, XO1=316, Rc=0)
     xorx = XS(31, XO1=316, Rc=1)
 
+    # Vector Ext
+
+    # floating point operations (ppc got it's own vector
+    # unit for double/single precision floating points
+
+    # FLOAT
+    # -----
+
+    # load
+    lxvdsx = XX1(31, XO1=332) # splat first element
+    lxvd2x = XX1(31, XO1=844)
+    lxvw4x = XX1(31, XO1=780)
+
+    # store
+    stxvd2x = XX1(31, XO1=972)
+    stxvw4x = XX1(31, XO1=908)
+
+    # arith
+
+    # add
+    xvadddp = XX3(60, XO9=96)
+    xvaddsp = XX3(60, XO9=64)
+    xsadddp = XX3(60, XO9=32)
+    # sub
+    xvsubdp = XX3(60, XO9=104)
+    xvsubsp = XX3(60, XO9=72)
+    # mul
+    xvmuldp = XX3(60, XO9=112)
+    xvmulsp = XX3(60, XO9=80)
+    xsmuldp = XX3(60, XO9=48)
+    # div
+    xvdivdp = XX3(60, XO9=102)
+    xvdivsp = XX3(60, XO9=88)
+    # cmp
+    xvcmpeqdp = XX3_2(60, XO11=99, OE=0)
+    xvcmpeqdpx = XX3_2(60, XO11=99, OE=1)
+    xvcmpeqsp = XX3_2(60, XO11=67, OE=0)
+    xvcmpeqspx = XX3_2(60, XO11=67, OE=1)
+
+    # logical and and complement
+    xxlandc = XX3(60, XO9=138)
+
+    # neg
+    xvnegdp = XX2(60, XO6=505)
+    xvnegsp = XX2(60, XO6=441)
+
+    # abs
+    xvabsdp = XX2(60, XO6=473)
+    xvabssp = XX2(60, XO6=409)
+
+    # conversion from/to
+    xvcvsxddp = XX2(60, XO6=504)
+    xvcvdpsxds = XX2(60, XO6=472)
+
+    # compare greater than unsigned int
+    vcmpgtubx = VC(4, XO12=518, OE=1)
+    vcmpgtub = VC(4, XO12=518, OE=0)
+    vcmpgtuhx = VC(4, XO12=584, OE=1)
+    vcmpgtuh = VC(4, XO12=584, OE=0)
+    vcmpgtuwx = VC(4, XO12=646, OE=1)
+    vcmpgtuw = VC(4, XO12=646, OE=0)
+    vcmpgtudx = VC(4, XO12=711, OE=1)
+    vcmpgtud = VC(4, XO12=711, OE=0)
+
+    # compare equal to unsigned int
+    vcmpequbx = VC(4, XO12=6, OE=1)
+    vcmpequb = VC(4, XO12=6, OE=0)
+    vcmpequhx = VC(4, XO12=70, OE=1)
+    vcmpequh = VC(4, XO12=70, OE=0)
+    vcmpequwx = VC(4, XO12=134, OE=1)
+    vcmpequw = VC(4, XO12=134, OE=0)
+    vcmpequdx = VC(4, XO12=199, OE=1)
+    vcmpequd = VC(4, XO12=199, OE=0)
+
+    # permute/splat
+    # splat low of A, and low of B
+    xxspltdl = XX3_splat(60, XO13=10, OE=0, DM=0b00)
+    # splat high of A, and high of B
+    xxspltdh = XX3_splat(60, XO13=10, OE=0, DM=0b11)
+    # generic splat
+    xxpermdi = XX3_splat(60, XO13=10, OE=0)
+
+    xxlxor = XX3(60, XO9=154)
+    xxlor = XX3(60, XO9=146)
+
+    # vector move register is alias to vector or
+    xvmr = xxlor
+
+    # INTEGER
+    # -------
+
+    # load
+    lvx = XV(31, XO1=103)
+    lvewx = XV(31, XO1=71)
+    lvehx = XV(31, XO1=39)
+    lvebx = XV(31, XO1=7)
+    # store
+    stvx = XV(31, XO1=231)
+    stvewx = XV(31, XO1=199)
+    stvehx = XV(31, XO1=167)
+    stvebx = XV(31, XO1=135)
+
+    # arith
+    vaddudm = VX(4, XO8=192)
+    vadduwm = VX(4, XO8=128)
+    vadduhm = VX(4, XO8=64)
+    vaddubm = VX(4, XO8=0)
+
+    vsubudm = VX(4, XO8=1216)
+    vsubuwm = VX(4, XO8=1152)
+    vsubuhm = VX(4, XO8=1088)
+    vsububm = VX(4, XO8=1024)
+
+    # logic
+    vand = VX(4, XO8=1028)
+    vor = VX(4, XO8=1156)
+    veqv = VX(4, XO8=1668)
+    vxor = VX(4, XO8=1220)
+    vnor = VX(4, XO8=1284)
+
+    # vector move register is alias to vector or
+    vmr = vor
+    # complement is equivalent to vnor
+    vnot = vnor
+
+    # shift, perm and select
+    lvsl = XV(31, XO1=6)
+    lvsr = XV(31, XO1=38)
+    vperm = VA(4, XO10=43)
+    vsel = VA(4, XO10=42)
+    vspltisb = VXI(4, XO8=780)
+    vspltisw = VXI(4, XO8=844)
+    vspltisw = VXI(4, XO8=908)
+
+    VX_splat = Form("ivrT", "ivrB", "ivrA", "XO8")
+    vspltb = VX_splat(4, XO8=524)
+    vsplth = VX_splat(4, XO8=588)
+    vspltw = VX_splat(4, XO8=652)
+
 class PPCAssembler(BasicPPCAssembler):
     BA = BasicPPCAssembler
 
diff --git a/rpython/jit/backend/ppc/condition.py b/rpython/jit/backend/ppc/condition.py
--- a/rpython/jit/backend/ppc/condition.py
+++ b/rpython/jit/backend/ppc/condition.py
@@ -6,6 +6,10 @@
 GE = 5
 SO = 6
 NS = 7
+VEQ = 8
+VEQI = 9
+VNE = 10
+VNEI = 11
 cond_none = -1    # invalid
 
 def negate(cond):
@@ -19,6 +23,8 @@
 assert negate(GE) == LT
 assert negate(SO) == NS
 assert negate(NS) == SO
+assert negate(VEQ) == VEQI
+assert negate(VNE) == VNEI
 
 encoding = [
     (2, 12),   # EQ
@@ -29,4 +35,8 @@
     (0, 4),    # GE
     (3, 12),   # SO
     (3, 4),    # NS
+    (24, 12),   # VEQ
+    (24, 4),    # VEQI
+    (26, 12),    # VNE
+    (26, 4),    # VNEI
 ]
diff --git a/rpython/jit/backend/ppc/detect_feature.py b/rpython/jit/backend/ppc/detect_feature.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/detect_feature.py
@@ -0,0 +1,46 @@
+import os
+import sys
+import struct
+import platform
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.tool import rffi_platform
+from rpython.rlib.rmmap import alloc, free
+from rpython.rlib.rstruct.runpack import runpack
+
+AT_HWCAP = rffi_platform.getconstantinteger('AT_HWCAP', '#include "linux/auxvec.h"')
+AT_NULL = rffi_platform.getconstantinteger('AT_NULL', '#include "linux/auxvec.h"')
+PPC_FEATURE_HAS_ALTIVEC = rffi_platform.getconstantinteger('PPC_FEATURE_HAS_ALTIVEC',
+                                                   '#include "asm/cputable.h"')
+SYSTEM = platform.system()
+
+def detect_vsx_linux():
+    try:
+        fd = os.open("/proc/self/auxv", os.O_RDONLY, 0644)
+        try:
+            while True:
+                buf = os.read(fd, 8)
+                buf2 = os.read(fd, 8)
+                if not buf or not buf2:
+                    break
+                key = runpack("L", buf)
+                value = runpack("L", buf2)
+                if key == AT_HWCAP:
+                    if value & PPC_FEATURE_HAS_ALTIVEC:
+                        return True
+                if key == AT_NULL:
+                    return False
+        finally:
+            os.close(fd)
+    except OSError:
+        pass
+    return False
+
+def detect_vsx():
+    if SYSTEM == 'Linux':
+        return detect_vsx_linux()
+    return False
+
+if __name__ == '__main__':
+    print 'The following extensions are supported:'
+    if detect_vsx():
+        print '  - AltiVec'
diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py
--- a/rpython/jit/backend/ppc/locations.py
+++ b/rpython/jit/backend/ppc/locations.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.history import INT, FLOAT
+from rpython.jit.metainterp.history import INT, FLOAT, VECTOR
 import sys
 
 # cannot import from arch.py, currently we have a circular import
@@ -30,6 +30,9 @@
     def is_fp_reg(self):
         return False
 
+    def is_vector_reg(self):
+        return False
+
     def is_imm_float(self):
         return False
 
@@ -75,6 +78,27 @@
     def as_key(self):
         return self.value + 100
 
+class VectorRegisterLocation(AssemblerLocation):
+    _immutable_ = True
+    width = WORD * 2
+    type = VECTOR
+
+    def __init__(self, value):
+        self.value = value
+
+    def __repr__(self):
+        return 'vr%d' % self.value
+
+    def is_reg(self):
+        return True
+
+    def as_key(self):
+        return self.value + 132
+
+    def is_vector_reg(self):
+        return True
+
+
 class ImmLocation(AssemblerLocation):
     _immutable_ = True
     width = WORD
@@ -129,9 +153,6 @@
     def __repr__(self):
         return 'FP(%s)+%d' % (self.type, self.value)
 
-    def location_code(self):
-        return 'b'
-
     def get_position(self):
         return self.position
 
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -29,7 +29,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.backend.ppc import callbuilder
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.backend.ppc.vector_ext import VectorAssembler
 
 class IntOpAssembler(object):
         
@@ -1329,7 +1329,8 @@
                   MiscOpAssembler, FieldOpAssembler,
                   StrOpAssembler, CallOpAssembler,
                   UnicodeOpAssembler, ForceOpAssembler,
-                  AllocOpAssembler, FloatOpAssembler):
+                  AllocOpAssembler, FloatOpAssembler,
+                  VectorAssembler):
     _mixin_ = True
 
     def nop(self):
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -14,6 +14,7 @@
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg
 import rpython.jit.backend.ppc.register as r
 import rpython.jit.backend.ppc.condition as c
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.backend.ppc.register import JITFRAME_FIXED_SIZE
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.jit.backend.llsupport import jitframe, rewrite
@@ -37,6 +38,7 @@
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rlib.rjitlog import rjitlog as jl
+from rpython.jit.backend.ppc.jump import remap_frame_layout_mixed
 
 memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address,
                                        rffi.SIZE_T], lltype.Void,
@@ -769,7 +771,7 @@
         self.update_frame_depth(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
         #
         size_excluding_failure_stuff = self.mc.get_relative_pos()
-        self.write_pending_failure_recoveries()
+        self.write_pending_failure_recoveries(regalloc)
         full_size = self.mc.get_relative_pos()
         #
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
@@ -813,8 +815,10 @@
         #    name = "Loop # %s: %s" % (looptoken.number, loopname)
         #    self.cpu.profile_agent.native_code_written(name,
         #                                               rawstart, full_size)
+        #print(hex(rawstart))
+        #import pdb; pdb.set_trace()
         return AsmInfo(ops_offset, rawstart + looppos,
-                       size_excluding_failure_stuff - looppos)
+                       size_excluding_failure_stuff - looppos, rawstart + looppos)
 
     def _assemble(self, regalloc, inputargs, operations):
         self._regalloc = regalloc
@@ -855,10 +859,12 @@
         self.reserve_gcref_table(allgcrefs)
         startpos = self.mc.get_relative_pos()
 
+        self._update_at_exit(arglocs, inputargs, faildescr, regalloc)
+
         self._check_frame_depth(self.mc, regalloc.get_gcmap())
         frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
         codeendpos = self.mc.get_relative_pos()
-        self.write_pending_failure_recoveries()
+        self.write_pending_failure_recoveries(regalloc)
         fullsize = self.mc.get_relative_pos()
         #
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
@@ -886,7 +892,8 @@
         self.fixup_target_tokens(rawstart)
         self.update_frame_depth(frame_depth)
         self.teardown()
-        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
+        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos,
+                       startpos + rawstart)
 
     def reserve_gcref_table(self, allgcrefs):
         # allocate the gc table right now.  We write absolute loads in
@@ -940,7 +947,7 @@
         ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
         mc.store(r.SCRATCH.value, r.SPP.value, ofs)
 
-    def break_long_loop(self):
+    def break_long_loop(self, regalloc):
         # If the loop is too long, the guards in it will jump forward
         # more than 32 KB.  We use an approximate hack to know if we
         # should break the loop here with an unconditional "b" that
@@ -948,15 +955,20 @@
         jmp_pos = self.mc.currpos()
         self.mc.trap()
 
-        self.write_pending_failure_recoveries()
+        self.write_pending_failure_recoveries(regalloc)
 
         currpos = self.mc.currpos()
         pmc = OverwritingBuilder(self.mc, jmp_pos, 1)
         pmc.b(currpos - jmp_pos)
         pmc.overwrite()
 
-    def generate_quick_failure(self, guardtok):
+    def generate_quick_failure(self, guardtok, regalloc):
         startpos = self.mc.currpos()
+        # accum vecopt
+        self._update_at_exit(guardtok.fail_locs, guardtok.failargs,
+                             guardtok.faildescr, regalloc)
+        pos = self.mc.currpos()
+        guardtok.rel_recovery_prefix = pos - startpos
         faildescrindex, target = self.store_info_on_descr(startpos, guardtok)
         assert target != 0
         self.mc.load_imm(r.r2, target)
@@ -969,13 +981,13 @@
             self.mc.trap()
         return startpos
 
-    def write_pending_failure_recoveries(self):
+    def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
         # at the end of self.mc.
         for i in range(self.pending_guard_tokens_recovered,
                        len(self.pending_guard_tokens)):
             tok = self.pending_guard_tokens[i]
-            tok.pos_recovery_stub = self.generate_quick_failure(tok)
+            tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
         self.pending_guard_tokens_recovered = len(self.pending_guard_tokens)
 
     def patch_pending_failure_recoveries(self, rawstart):
@@ -986,7 +998,7 @@
             addr = rawstart + tok.pos_jump_offset
             #
             # XXX see patch_jump_for_descr()
-            tok.faildescr.adr_jump_offset = rawstart + tok.pos_recovery_stub
+            tok.faildescr.adr_jump_offset = rawstart + tok.pos_recovery_stub + tok.rel_recovery_prefix
             #
             relative_target = tok.pos_recovery_stub - tok.pos_jump_offset
             #
@@ -1058,6 +1070,10 @@
                 self.mc.lfd(reg, r.SPP.value, offset)
                 return
             assert 0, "not supported location"
+        elif prev_loc.is_vector_reg():
+            assert loc.is_vector_reg()
+            self.mc.vmr(loc.value, prev_loc.value, prev_loc.value)
+            return
         elif prev_loc.is_reg():
             reg = prev_loc.value
             # move to another register
@@ -1363,6 +1379,62 @@
             self.mc.load_imm(r.SCRATCH, fail_index)
             self.mc.store(r.SCRATCH.value, r.SPP.value, FORCE_INDEX_OFS)
 
+    def stitch_bridge(self, faildescr, target):
+        """ Stitching means that one can enter a bridge with a complete different register
+            allocation. This needs remapping which is done here for both normal registers
+            and accumulation registers.
+        """
+        asminfo, bridge_faildescr, version, looptoken = target
+        assert isinstance(bridge_faildescr, ResumeGuardDescr)
+        assert isinstance(faildescr, ResumeGuardDescr)
+        assert asminfo.rawstart != 0
+        self.mc = PPCBuilder()
+        allblocks = self.get_asmmemmgr_blocks(looptoken)
+        self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
+                                                   allblocks)
+        frame_info = self.datablockwrapper.malloc_aligned(
+            jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
+
+        # if accumulation is saved at the guard, we need to update it here!
+        guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs)
+        bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs)
+        guard_accum_info = faildescr.rd_vector_info
+        # O(n**2), but usually you only have at most 1 fail argument
+        while guard_accum_info:
+            bridge_accum_info = bridge_faildescr.rd_vector_info
+            while bridge_accum_info:
+                if bridge_accum_info.failargs_pos == guard_accum_info.failargs_pos:
+                    # the mapping might be wrong!
+                    if bridge_accum_info.location is not guard_accum_info.location:
+                        self.regalloc_mov(guard_accum_info.location, bridge_accum_info.location)
+                bridge_accum_info = bridge_accum_info.next()
+            guard_accum_info = guard_accum_info.next()
+
+        # register mapping is most likely NOT valid, thus remap it
+        src_locations1 = []
+        dst_locations1 = []
+        src_locations2 = []
+        dst_locations2 = []
+
+        # Build the four lists
+        assert len(guard_locs) == len(bridge_locs)
+        for i,src_loc in enumerate(guard_locs):
+            dst_loc = bridge_locs[i]
+            if not src_loc.is_fp_reg():
+                src_locations1.append(src_loc)
+                dst_locations1.append(dst_loc)
+            else:
+                src_locations2.append(src_loc)
+                dst_locations2.append(dst_loc)
+        remap_frame_layout_mixed(self, src_locations1, dst_locations1, r.SCRATCH,
+                                 src_locations2, dst_locations2, r.FP_SCRATCH)
+
+        offset = self.mc.get_relative_pos()
+        self.mc.b_abs(asminfo.rawstart)
+
+        rawstart = self.materialize_loop(looptoken)
+        # update the guard to jump right to this custom piece of assembler
+        self.patch_jump_for_descr(faildescr, rawstart)
 
 def notimplemented_op(self, op, arglocs, regalloc):
     msg = '[PPC/asm] %s not implemented\n' % op.getopname()
diff --git a/rpython/jit/backend/ppc/ppc_field.py b/rpython/jit/backend/ppc/ppc_field.py
--- a/rpython/jit/backend/ppc/ppc_field.py
+++ b/rpython/jit/backend/ppc/ppc_field.py
@@ -43,6 +43,17 @@
     "spr":    (11, 20),
     "TO":     ( 6, 10),
     "UIMM":   (16, 31),
+    "fvrT":   (6,  31, 'unsigned', regname._V, 'overlap'),
+    "fvrA":   (11, 29, 'unsigned', regname._V, 'overlap'),
+    "fvrB":   (16, 30, 'unsigned', regname._V, 'overlap'),
+    # low vector register T (low in a sense:
+    # can only address 32 vector registers)
+    "ivrT":   (6,  10, 'unsigned', regname._V),
+    # low vector register A
+    "ivrA":   (11, 15, 'unsigned', regname._V),
+    # low vector register B
+    "ivrB":   (16, 20, 'unsigned', regname._V),
+    "ivrC":   (21, 25, 'unsigned', regname._V),
     "XO1":    (21, 30),
     "XO2":    (22, 30),
     "XO3":    (26, 30),
@@ -50,7 +61,15 @@
     "XO5":    (27, 29),
     "XO6":    (21, 29),
     "XO7":    (27, 30),
+    "XO8":    (21, 31),
+    "XO9":    (21, 28),
+    "XO10":   (26, 31),
+    "XO11":   (22, 28),
+    "XO12":   (22, 31),
+    "XO13":   (24, 28),
+    "DM":     (22, 23),
     "LL":     ( 9, 10),
+    "SIM":    (11, 15),
 }
 
 
@@ -100,7 +119,6 @@
     def decode(self, inst):
         value = super(sh, self).decode(inst)
         return (value & 32) << 5 | (value >> 10 & 31)
-# other special fields?
 
 ppc_fields = {
     "LI":  IField("LI", *fields["LI"]),
diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py
--- a/rpython/jit/backend/ppc/rassemblermaker.py
+++ b/rpython/jit/backend/ppc/rassemblermaker.py
@@ -46,6 +46,15 @@
         elif field.name == 'sh':
             body.append('sh1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value))
             value = 'sh1'
+        elif field.name == 'fvrT':
+            body.append('vrT1 = (%s & 31) << 21 | (%s & 32) >> 5' % (value, value))
+            value = 'vrT1'
+        elif field.name == 'fvrA':
+            body.append('fvrA1 = (%s & 31) << 14 | (%s & 32) >> 5' % (value, value))
+            value = 'fvrA1'
+        elif field.name == 'fvrB':
+            body.append('fvrB1 = (%s & 31) << 10 | (%s & 32) >> 5' % (value, value))
+            value = 'fvrB1'
         if isinstance(field, IField):
             body.append('v |= ((%3s >> 2) & r_uint(%#05x)) << 2' % (value, field.mask))
         else:
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -10,7 +10,8 @@
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg, check_imm_box
 from rpython.jit.backend.ppc.helper import regalloc as helper
 from rpython.jit.metainterp.history import (Const, ConstInt, ConstFloat, ConstPtr,
-                                            INT, REF, FLOAT, VOID)
+                                            INT, REF, FLOAT, VOID, VECTOR,
+                                            AbstractFailDescr)
 from rpython.jit.metainterp.history import JitCellToken, TargetToken
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.ppc import locations
@@ -27,6 +28,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.rlib import rgc
 from rpython.rlib.rarithmetic import r_uint
+from rpython.jit.backend.ppc.vector_ext import VectorRegalloc
 
 LIMIT_LOOP_BREAK = 15000      # should be much smaller than 32 KB
 
@@ -49,6 +51,11 @@
     def __repr__(self):
         return "<TempFloat at %s>" % (id(self),)
 
+class TempVector(TempVar):
+    type = VECTOR
+
+    def __repr__(self):
+        return "<TempVector at %s>" % (id(self),)
 
 class FPRegisterManager(RegisterManager):
     all_regs              = r.MANAGED_FP_REGS
@@ -135,6 +142,26 @@
         self.temp_boxes.append(box)
         return reg
 
+class VectorRegisterManager(RegisterManager):
+    all_regs              = r.MANAGED_FLOAT_VECTOR_REGS
+    box_types             = [FLOAT, INT]
+    save_around_call_regs = [] # ??? lookup the ABI
+    assert set(save_around_call_regs).issubset(all_regs)
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        RegisterManager.__init__(self, longevity, frame_manager, assembler)
+
+    def ensure_reg(self, box):
+        raise NotImplementedError
+
+    def get_scratch_reg(self, type=INT):
+        if type == FLOAT:
+            box = TempFloat()
+        else:
+            box = TempInt()
+        reg = self.force_allocate_reg(box, forbidden_vars=self.temp_boxes)
+        self.temp_boxes.append(box)
+        return reg
 
 class PPCFrameManager(FrameManager):
     def __init__(self, base_ofs):
@@ -155,8 +182,7 @@
         assert isinstance(loc, locations.StackLocation)
         return loc.position
 
-
-class Regalloc(BaseRegalloc):
+class Regalloc(BaseRegalloc, VectorRegalloc):
 
     def __init__(self, assembler=None):
         self.cpu = assembler.cpu
@@ -180,6 +206,8 @@
                                      assembler = self.assembler)
         self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm,
                                       assembler = self.assembler)
+        self.vrm = VectorRegisterManager(self.longevity, frame_manager = self.fm,
+                                         assembler = self.assembler)
         return operations
 
     def prepare_loop(self, inputargs, operations, looptoken, allgcrefs):
@@ -241,7 +269,10 @@
 
     def possibly_free_var(self, var):
         if var is not None:
-            if var.type == FLOAT:
+            if var.is_vector():
+                if var.type != VOID:
+                    self.vrm.possibly_free_var(var)
+            elif var.type == FLOAT:
                 self.fprm.possibly_free_var(var)
             else:
                 self.rm.possibly_free_var(var)
@@ -287,6 +318,7 @@
             self.assembler.mc.mark_op(op)
             self.rm.position = i
             self.fprm.position = i
+            self.vrm.position = i
             opnum = op.opnum
             if rop.has_no_side_effect(opnum) and op not in self.longevity:
                 i += 1
@@ -295,7 +327,10 @@
             #
             for j in range(op.numargs()):
                 box = op.getarg(j)
-                if box.type != FLOAT:
+                if box.is_vector():
+                    if box.type != VOID:
+                        self.vrm.temp_boxes.append(box)
+                elif box.type != FLOAT:
                     self.rm.temp_boxes.append(box)
                 else:
                     self.fprm.temp_boxes.append(box)
@@ -309,8 +344,9 @@
             self.possibly_free_var(op)
             self.rm._check_invariants()
             self.fprm._check_invariants()
+            self.vrm._check_invariants()
             if self.assembler.mc.get_relative_pos() > self.limit_loop_break:
-                self.assembler.break_long_loop()
+                self.assembler.break_long_loop(self)
                 self.limit_loop_break = (self.assembler.mc.get_relative_pos() +
                                              LIMIT_LOOP_BREAK)
             i += 1
@@ -351,10 +387,13 @@
         return gcmap
 
     def loc(self, var):
-        if var.type == FLOAT:
-            return self.fprm.loc(var)
+        if var.is_vector():
+            return self.vrm.loc(var)
         else:
-            return self.rm.loc(var)
+            if var.type == FLOAT:
+                return self.fprm.loc(var)
+            else:
+                return self.rm.loc(var)
 
     def next_instruction(self):
         self.rm.next_instruction()
@@ -419,6 +458,7 @@
         # temporary boxes and all the current operation's arguments
         self.rm.free_temp_vars()
         self.fprm.free_temp_vars()
+        self.vrm.free_temp_vars()
 
     # ******************************************************
     # *         P R E P A R E  O P E R A T I O N S         * 
@@ -550,7 +590,19 @@
         #
         # generate_quick_failure() produces up to 14 instructions per guard
         self.limit_loop_break -= 14 * 4
-        #
+        # specifically for vecopt
+        descr = op.getdescr()
+        if not descr:
+            return args
+        assert isinstance(descr, AbstractFailDescr)
+        if descr.rd_vector_info:
+            accuminfo = descr.rd_vector_info
+            while accuminfo:
+                i = accuminfo.getpos_in_failargs()+1
+                accuminfo.location = args[i]
+                loc = self.loc(accuminfo.getoriginal())
+                args[i] = loc
+                accuminfo = accuminfo.next()
         return args
 
     def load_condition_into_cc(self, box):
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -1,8 +1,10 @@
 from rpython.jit.backend.ppc.locations import (RegisterLocation,
-                                               FPRegisterLocation)
+       FPRegisterLocation, VectorRegisterLocation)
 
 ALL_REGS        = [RegisterLocation(i) for i in range(32)]
 ALL_FLOAT_REGS  = [FPRegisterLocation(i) for i in range(32)]
+ALL_INTEGER_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(32)]
+ALL_FLOAT_VECTOR_REGS = [VectorRegisterLocation(i) for i in range(32,64)]
 
 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15, r16,\
     r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31\
@@ -12,6 +14,19 @@
     f17, f18, f19, f20, f21, f22, f23, f24, f25, f26, f27, f28, f29, f30, f31\
     = ALL_FLOAT_REGS
 
+ivr0, ivr1, ivr2, ivr3, ivr4, ivr5, ivr6, ivr7, ivr8, ivr9, ivr10, ivr11, ivr12,\
+    ivr13, ivr14, ivr15, ivr16, ivr17, ivr18, ivr19, ivr20, ivr21, ivr22, ivr23,\
+    ivr24, ivr25, ivr26, ivr27, ivr28, ivr29, ivr30, ivr31\
+    = ALL_FLOAT_REGS
+
+# the first 32 vector register are partly shared with the normal floating point
+# registers, since there are so many registers, we just take the upper 31 ones
+vr32, vr33, vr34, vr35, vr36, vr37, \
+      vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \
+      vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \
+      vr61, vr62, vr63 = ALL_FLOAT_VECTOR_REGS
+
+
 NONVOLATILES        = [r14, r15, r16, r17, r18, r19, r20, r21, r22, r23,
                     r24, r25, r26, r27, r28, r29, r30, r31]
 VOLATILES           = [r3, r4, r5, r6, r7, r8, r9, r10, r11, r12]
@@ -43,6 +58,9 @@
 
 MANAGED_FP_REGS = VOLATILES_FLOAT #+ NONVOLATILES_FLOAT
 
+MANAGED_FLOAT_VECTOR_REGS = ALL_FLOAT_VECTOR_REGS
+MANAGED_INTEGER_VECTOR_REGS = ALL_INTEGER_VECTOR_REGS
+
 assert RCS1 in MANAGED_REGS and RCS1 in NONVOLATILES
 assert RCS2 in MANAGED_REGS and RCS2 in NONVOLATILES
 assert RCS3 in MANAGED_REGS and RCS3 in NONVOLATILES
diff --git a/rpython/jit/backend/ppc/regname.py b/rpython/jit/backend/ppc/regname.py
--- a/rpython/jit/backend/ppc/regname.py
+++ b/rpython/jit/backend/ppc/regname.py
@@ -6,6 +6,10 @@
     def __repr__(self):
         return "fr%s"%(super(_F, self).__repr__(),)
     __str__ = __repr__
+class _V(int):
+    def __repr__(self):
+        return "vr%s"%(super(_V, self).__repr__(),)
+    __str__ = __repr__
 
 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, \
     r13, r14, r15, r16, r17, r18, r19, r20, r21, r22, \
@@ -15,4 +19,11 @@
      fr13, fr14, fr15, fr16, fr17, fr18, fr19, fr20, fr21, fr22, \
      fr23, fr24, fr25, fr26, fr27, fr28, fr29, fr30, fr31 = map(_F, range(32))
 
+vr0, vr1, vr2, vr3, vr4, vr5, vr6, vr7, vr8, vr9, vr10, vr11, vr12, vr13, \
+     vr14, vr15, vr16, vr17, vr18, vr19, vr20, vr21, vr22, vr23, vr24, vr25, \
+     vr26, vr27, vr28, vr29, vr30, vr31, vr32, vr33, vr34, vr35, vr36, vr37, \
+     vr38, vr39, vr40, vr41, vr42, vr43, vr44, vr45, vr46, vr47, vr48, \
+     vr49, vr50, vr51, vr52, vr53, vr54, vr55, vr56, vr57, vr58, vr59, vr60, \
+     vr61, vr62, vr63 = map(_V, range(64))
+
 crf0, crf1, crf2, crf3, crf4, crf5, crf6, crf7 = range(8)
diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -3,13 +3,15 @@
 from rpython.rlib import rgc
 from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER
 from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
+from rpython.jit.backend.ppc.vector_ext import AltiVectorExt
 from rpython.jit.backend.ppc.ppc_assembler import AssemblerPPC
 from rpython.jit.backend.ppc.arch import WORD
 from rpython.jit.backend.ppc.codebuilder import PPCBuilder
 from rpython.jit.backend.ppc import register as r
 
+class PPC_CPU(AbstractLLCPU):
 
-class PPC_CPU(AbstractLLCPU):
+    vector_ext = AltiVectorExt()
 
     supports_floats = True
     # missing: supports_singlefloats
diff --git a/rpython/jit/backend/ppc/test/test_ppcvector.py b/rpython/jit/backend/ppc/test/test_ppcvector.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/test/test_ppcvector.py
@@ -0,0 +1,26 @@
+import py
+from rpython.jit.backend.ppc.test import test_basic
+from rpython.jit.metainterp.test import test_vector
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
+
+
+class TestBasic(test_basic.JitPPCMixin, test_vector.VectorizeTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_basic.py
+    def setup_method(self, method):
+        clazz = self.CPUClass
+        def init(*args, **kwargs):
+            cpu = clazz(*args, **kwargs)
+            # > 95% can be executed, thus let's cheat here a little
+            cpu.supports_guard_gc_type = True
+            return cpu
+        self.CPUClass = init
+
+    def supports_vector_ext(self):
+        return detect_vsx()
+
+    def test_list_vectorize(self):
+        pass # needs support_guard_gc_type, disable for now
+
+    enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
+
diff --git a/rpython/jit/backend/ppc/test/test_vector_instr.py b/rpython/jit/backend/ppc/test/test_vector_instr.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/test/test_vector_instr.py
@@ -0,0 +1,95 @@
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+from rpython.jit.backend.ppc.codebuilder import BasicPPCAssembler, PPCBuilder
+from rpython.jit.backend.ppc.regname import *
+from rpython.jit.backend.ppc.register import *
+from rpython.jit.backend.ppc import form
+from rpython.jit.backend import detect_cpu
+from rpython.jit.backend.ppc.arch import IS_PPC_32, IS_PPC_64, IS_BIG_ENDIAN
+from rpython.jit.backend.ppc.arch import WORD
+
+cpu = detect_cpu.autodetect()
+
+signed = lltype.Signed
+unsigned = lltype.Unsigned
+char = lltype.Char
+
+def vec_asmtest(memory=[]):
+    def testmaker(test):
+        def newtest(self):
+            memory_ptrs = []
+            a = PPCBuilder()
+            for (bytes, type, values) in memory:
+                # alloc
+                adr = lltype.malloc(rffi.CArray(char), bytes, flavor="raw")
+                memory_ptrs.append(adr)
+                address = adr
+                for i,value in enumerate(values):
+                    rffi.cast(rffi.CArrayPtr(type), adr)[i] = rffi.cast(type, value)
+
+            expected = test(self, a, *[rffi.cast(lltype.Signed, m) for m in memory_ptrs])
+            f = a.get_assembler_function()
+            f()
+            for expect, type, ptr in expected:
+                value = rffi.cast(rffi.CArrayPtr(type), ptr)[0]
+                assert value == expect
+
+            while memory_ptrs:
+                ptr = memory_ptrs.pop()
+                lltype.free(ptr, flavor="raw")
+        return newtest
+    return testmaker
+
+
+class TestAssemble(object):
+    """
+    See how tests are built in test_ppc.py
+
+    Instead of asmtest, use vec_asmtest. It adds a parameter 'memory' that
+    allocates memory.
+
+
+    @vec_asmtest(memory=[(8, [1,2]), 'm2': (16, [1,2,3,4])])
+    def test(self, builder, m, m2):
+        # ...
+        return [ (value, type, ptr), ... ]
+    """
+    def setup_class(cls):
+        if cpu not in ["ppc", "ppc64", "ppc-64"]:
+            py.test.skip("can't test all of ppcgen on non-PPC!")
+
+    @vec_asmtest(memory=[(16, signed, [0,0])])
+    def test_unaligned_load(self, a, mem):
+        a.load_imm(r15, mem)
+        a.lxvd2x(0, 0, r15.value)
+        a.blr()
+        return [ (0, signed, mem), (0, signed, mem+8) ]
+
+    @vec_asmtest(memory=[(16, signed, [1,2]), (16, signed, [0,0])])
+    def test_unaligned_load_and_store(self, a, mem_l, mem_t):
+        a.load_imm(r15, mem_l)
+        a.load_imm(r14, mem_t)
+        a.lxvd2x(0, 0, r15.value)
+        a.stxvd2x(0, 0, r14.value)
+        a.blr()
+        return [ (1, signed, mem_t), (2, signed, mem_t+8) ]
+
+    def test_xx3_instr(self):
+        a = PPCBuilder()
+        def assign_to_self(v):
+            self.last_value = v
+        a.emit = assign_to_self
+
+        a.xxspltdl(32, 32, 32)
+        #                                               tttttaaaaabbbbb        abt
+        assert hex(int(self.last_value)) == hex(0b11110000000000000000000001010111)
+        a.xxspltdl(32, 2, 2)
+        #                                               tttttaaaaabbbbb        abt
+        assert hex(int(self.last_value)) == hex(0b11110000000000100001000001010001)
+        a.xxspltdl(0, 63, 0)
+        #                                               tttttaaaaabbbbb        abt
+        assert hex(int(self.last_value)) == hex(0b11110000000111110000000001010100)
+        a.xxspltdl(0, 0, 63)
+        #                                               tttttaaaaabbbbb        abt
+        assert hex(int(self.last_value)) == hex(0b11110000000000001111100001010010)
+
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -0,0 +1,871 @@
+import py
+from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.history import (ConstInt, INT, REF,
+    FLOAT, VECTOR, TargetToken)
+from rpython.jit.backend.llsupport.descr import (ArrayDescr, CallDescr,
+    unpack_arraydescr, unpack_fielddescr, unpack_interiorfielddescr)
+from rpython.jit.backend.llsupport.regalloc import get_scale
+from rpython.jit.metainterp.resoperation import (rop, ResOperation,
+        VectorOp, VectorGuardOp)
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem import lltype
+from rpython.jit.backend.ppc.locations import imm, RegisterLocation
+from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
+from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET, WORD
+import rpython.jit.backend.ppc.register as r
+import rpython.jit.backend.ppc.condition as c
+import rpython.jit.backend.ppc.locations as l
+from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.jit.codewriter import longlong
+from rpython.jit.backend.ppc.detect_feature import detect_vsx
+from rpython.rlib.objectmodel import always_inline
+
+def not_implemented(msg):
+    msg = '[ppc/vector_ext] %s\n' % msg
+    if we_are_translated():
+        llop.debug_print(lltype.Void, msg)
+    raise NotImplementedError(msg)
+
+ at always_inline
+def permi(v1, v2):
+    """ permute immediate for big and little endian """
+    # if v1 == 0 unpacks index 0 of param 1
+    # if v1 == 1 unpacks index 1 of param 1
+    # if v2 == 0 unpacks index 0 of param 2
+    # if v2 == 1 unpacks index 1 of param 2
+    mask = 0
+    if IS_BIG_ENDIAN:
+        not_implemented("no big endian support (yet)")
+    else:
+        if v1 == 0: mask |= 0b01
+        if v1 == 1: mask |= 0b00
+        if v2 == 0: mask |= 0b10
+        if v2 == 1: mask |= 0b00
+    return mask
+
+
+def flush_vec_cc(asm, regalloc, condition, size, result_loc):
+    # After emitting an instruction that leaves a boolean result in
+    # a condition code (cc), call this.  In the common case, result_loc
+    # will be set to SPP by the regalloc, which in this case means
+    # "propagate it between this operation and the next guard by keeping
+    # it in the cc".  In the uncommon case, result_loc is another
+    # register, and we emit a load from the cc into this register.
+
+    # Possibly invert the bit in the CR
+    bit, invert = c.encoding[condition]
+    assert 24 <= bit <= 27
+    if invert == 12:
+        pass
+    elif invert == 4:
+        asm.mc.crnor(bit, bit, bit)
+    else:
+        assert 0
+    assert asm.guard_success_cc == c.cond_none
+    #
+    if result_loc is r.SPP:
+        asm.guard_success_cc = condition
+    else:
+        resval = result_loc.value
+        # either doubleword integer 1 (2x) or word integer 1 (4x)
+        ones = regalloc.vrm.get_scratch_reg(type=INT).value
+        zeros = regalloc.vrm.get_scratch_reg(type=INT).value
+        asm.mc.vxor(zeros, zeros, zeros)
+        if size == 4:
+            asm.mc.vspltisw(ones, 1)
+        else:
+            assert size == 8
+            tloc = regalloc.rm.get_scratch_reg()
+            asm.mc.load_imm(tloc, asm.VEC_DOUBLE_WORD_ONES)
+            asm.mc.lvx(ones, 0, tloc.value)
+        asm.mc.vsel(resval, zeros, ones, resval)
+
+class AltiVectorExt(VectorExt):
+    def setup_once(self, asm):
+        if detect_vsx():
+            self.enable(16, accum=True)
+            asm.setup_once_vector()
+        self._setup = True
+
+class VectorAssembler(object):
+    _mixin_ = True
+
+    VEC_DOUBLE_WORD_ONES = 0
+
+    def setup_once_vector(self):
+        if IS_BIG_ENDIAN:
+            # 2x 64 bit signed integer(1) BE
+            data = (b'\x00' * 7 + b'\x01') * 2
+        else:
+            # 2x 64 bit signed integer(1) LE
+            data = (b'\x01' + b'\x00' * 7) * 2
+        datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, [])
+        mem = datablockwrapper.malloc_aligned(len(data), alignment=16)
+        datablockwrapper.done()
+        addr = rffi.cast(rffi.CArrayPtr(lltype.Char), mem)
+        for i in range(len(data)):
+            addr[i] = data[i]
+        self.VEC_DOUBLE_WORD_ONES = mem
+
+    def emit_vec_load_f(self, op, arglocs, regalloc):
+        resloc, baseloc, indexloc, size_loc, ofs, integer_loc = arglocs
+        indexloc = self._apply_offset(indexloc, ofs)
+        itemsize = size_loc.value
+        if integer_loc.value:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 4:
+            self.mc.lxvw4x(resloc.value, indexloc.value, baseloc.value)
+        elif itemsize == 8:
+            self.mc.lxvd2x(resloc.value, indexloc.value, baseloc.value)
+        else:
+            not_implemented("vec_load_f itemsize %d" % itemsize)
+
+    emit_vec_load_i = emit_vec_load_f
+
+    def emit_vec_store(self, op, arglocs, regalloc):
+        baseloc, indexloc, valueloc, sizeloc, baseofs, \
+            integer_loc = arglocs
+        indexloc = self._apply_offset(indexloc, baseofs)
+        assert baseofs.value == 0
+        if integer_loc.value:
+            self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+        else:
+            itemsize = sizeloc.value
+            if itemsize == 4:
+                self.mc.stxvw4x(valueloc.value, indexloc.value, baseloc.value)
+            elif itemsize == 8:
+                self.mc.stxvd2x(valueloc.value, indexloc.value, baseloc.value)
+            else:
+                not_implemented("vec_store itemsize %d" % itemsize)
+
+    def emit_vec_int_add(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            self.mc.vaddubm(resloc.value, loc0.value, loc1.value)
+        elif size == 2:
+            self.mc.vadduhm(resloc.value, loc0.value, loc1.value)
+        elif size == 4:
+            self.mc.vadduwm(resloc.value, loc0.value, loc1.value)
+        elif size == 8:
+            self.mc.vaddudm(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_sub(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, size_loc = arglocs
+        size = size_loc.value
+        if size == 1:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsububm(resloc.value, loc0.value, loc1.value)
+        elif size == 2:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuhm(resloc.value, loc0.value, loc1.value)
+        elif size == 4:
+            # TODO verify if unsigned subtract is the wanted feature
+            self.mc.vsubuwm(resloc.value, loc0.value, loc1.value)
+        elif size == 8:
+            self.mc.vsubudm(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_add(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvaddsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvadddp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_sub(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvsubsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvsubdp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_mul(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvmulsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvmuldp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_float_truediv(self, op, arglocs, regalloc):
+        resloc, loc0, loc1, itemsize_loc = arglocs
+        itemsize = itemsize_loc.value
+        if itemsize == 4:
+            self.mc.xvdivsp(resloc.value, loc0.value, loc1.value)
+        elif itemsize == 8:
+            self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
+
+    def emit_vec_int_mul(self, op, arglocs, regalloc):
+        raise NotImplementedError
+        pass # TODO
+
+    def emit_vec_int_and(self, op, arglocs, regalloc):

From pypy.commits at gmail.com  Thu Nov  3 04:37:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 01:37:55 -0700 (PDT)
Subject: [pypy-commit] pypy ppc-vsx-support: close branch
Message-ID: <581af763.11881c0a.b09d1.cc29@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: ppc-vsx-support
Changeset: r88098:e2f4c23003f3
Date: 2016-11-03 09:36 +0100
http://bitbucket.org/pypy/pypy/changeset/e2f4c23003f3/

Log:	close branch


From pypy.commits at gmail.com  Thu Nov  3 05:50:54 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:50:54 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: unicodedata
Message-ID: <581b087e.a59dc20a.4bce.6167@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88099:b2eced084549
Date: 2016-11-02 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/b2eced084549/

Log:	unicodedata

diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -24,13 +24,13 @@
         self.line_num = 0
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     @objectmodel.dont_inline
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
-        w_error = space.getattr(w_module, space.wrap('Error'))
+        w_error = space.getattr(w_module, space.newtext('Error'))
         raise oefmt(w_error, "line %d: %s", self.line_num, msg)
 
     def add_char(self, field_builder, c):
@@ -49,10 +49,10 @@
             try:
                 ff = string_to_float(field)
             except ParseStringError as e:
-                raise wrap_parsestringerror(space, e, space.wrap(field))
-            w_obj = space.wrap(ff)
+                raise wrap_parsestringerror(space, e, space.newtext(field))
+            w_obj = space.newfloat(ff)
         else:
-            w_obj = space.wrap(field)
+            w_obj = space.newtext(field)
         self.fields_w.append(w_obj)
 
     def next_w(self):
@@ -267,4 +267,4 @@
     old_limit = field_limit.limit
     if new_limit >= 0:
         field_limit.limit = new_limit
-    return space.wrap(old_limit)
+    return space.newint(old_limit)
diff --git a/pypy/module/binascii/interp_binascii.py b/pypy/module/binascii/interp_binascii.py
--- a/pypy/module/binascii/interp_binascii.py
+++ b/pypy/module/binascii/interp_binascii.py
@@ -7,8 +7,8 @@
 
 def raise_Error(space, msg):
     w_error = space.fromcache(Cache).w_error
-    raise OperationError(w_error, space.wrap(msg))
+    raise OperationError(w_error, space.newtext(msg))
 
 def raise_Incomplete(space, msg):
     w_error = space.fromcache(Cache).w_incomplete
-    raise OperationError(w_error, space.wrap(msg))
+    raise OperationError(w_error, space.newtext(msg))
diff --git a/pypy/module/binascii/interp_crc32.py b/pypy/module/binascii/interp_crc32.py
--- a/pypy/module/binascii/interp_crc32.py
+++ b/pypy/module/binascii/interp_crc32.py
@@ -9,4 +9,4 @@
 
     crc = rzipfile.crc32(data, r_uint(oldcrc))
     crc = rffi.cast(rffi.INT, crc)    # unsigned => 32-bit signed
-    return space.wrap(intmask(crc))
+    return space.newint(intmask(crc))
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -99,18 +99,18 @@
         try:
             code = self._lookup(name.upper())
         except KeyError:
-            msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name))
+            msg = space.mod(space.newtext("undefined character name '%s'"), space.newtext(name))
             raise OperationError(space.w_KeyError, msg)
-        return space.wrap(code)
+        return space.newint(code)
 
     @unwrap_spec(name=str)
     def lookup(self, space, name):
         try:
             code = self._lookup(name.upper())
         except KeyError:
-            msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name))
+            msg = space.mod(space.newtext("undefined character name '%s'"), space.newtext(name))
             raise OperationError(space.w_KeyError, msg)
-        return space.wrap(code_to_unichr(code))
+        return space.newunicode(code_to_unichr(code))
 
     def name(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
@@ -120,12 +120,12 @@
             if w_default is not None:
                 return w_default
             raise oefmt(space.w_ValueError, "no such name")
-        return space.wrap(name)
+        return space.newtext(name)
 
     def decimal(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._decimal(code))
+            return space.newint(self._decimal(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -135,7 +135,7 @@
     def digit(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._digit(code))
+            return space.newint(self._digit(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -145,7 +145,7 @@
     def numeric(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)
         try:
-            return space.wrap(self._numeric(code))
+            return space.newfloat(self._numeric(code))
         except KeyError:
             pass
         if w_default is not None:
@@ -154,28 +154,28 @@
 
     def category(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._category(code))
+        return space.newtext(self._category(code))
 
     def east_asian_width(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._east_asian_width(code))
+        return space.newtext(self._east_asian_width(code))
 
     def bidirectional(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._bidirectional(code))
+        return space.newtext(self._bidirectional(code))
 
     def combining(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._combining(code))
+        return space.newint(self._combining(code))
 
     def mirrored(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
         # For no reason, unicodedata.mirrored() returns an int, not a bool
-        return space.wrap(int(self._mirrored(code)))
+        return space.newint(int(self._mirrored(code)))
 
     def decomposition(self, space, w_unichr):
         code = unichr_to_code_w(space, w_unichr)
-        return space.wrap(self._decomposition(code))
+        return space.newtext(self._decomposition(code))
 
     @unwrap_spec(form=str)
     def normalize(self, space, form, w_unistr):
@@ -204,7 +204,7 @@
         resultlen = len(result)
         # Expand the character
         for i in range(strlen):
-            ch = space.int_w(space.ord(space.getitem(w_unistr, space.wrap(i))))
+            ch = space.int_w(space.ord(space.getitem(w_unistr, space.newint(i))))
             # Do Hangul decomposition
             if SBase <= ch < SBase + SCount:
                 SIndex = ch - SBase
@@ -259,10 +259,10 @@
                 result[0] = ch
 
         if not composed: # If decomposed normalization we are done
-            return space.wrap(u''.join([unichr(i) for i in result[:j]]))
+            return space.newunicode(u''.join([unichr(i) for i in result[:j]]))
 
         if j <= 1:
-            return space.wrap(u''.join([unichr(i) for i in result[:j]]))
+            return space.newunicode(u''.join([unichr(i) for i in result[:j]]))
 
         current = result[0]
         starter_pos = 0
@@ -309,7 +309,7 @@
 
         result[starter_pos] = current
 
-        return space.wrap(u''.join([unichr(i) for i in result[:next_insert]]))
+        return space.newunicode(u''.join([unichr(i) for i in result[:next_insert]]))
 
 
 methods = {}

From pypy.commits at gmail.com  Thu Nov  3 05:50:57 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:50:57 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: continuation
Message-ID: <581b0881.c6bdc20a.bc113.640f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88101:9ade4b7cdbd7
Date: 2016-11-03 08:06 +0100
http://bitbucket.org/pypy/pypy/changeset/9ade4b7cdbd7/

Log:	continuation

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -35,7 +35,7 @@
         w_args, w_kwds = __args__.topacked()
         bottomframe = space.createframe(get_entrypoint_pycode(space),
                                         get_w_module_dict(space), None)
-        bottomframe.locals_cells_stack_w[0] = space.wrap(self)
+        bottomframe.locals_cells_stack_w[0] = self
         bottomframe.locals_cells_stack_w[1] = w_callable
         bottomframe.locals_cells_stack_w[2] = w_args
         bottomframe.locals_cells_stack_w[3] = w_kwds
@@ -126,13 +126,13 @@
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
-    return space.wrap(r)
+    return r
 
 def unpickle(space, w_subtype):
     """Pickle support."""
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
-    return space.wrap(r)
+    return r
 
 
 W_Continulet.typedef = TypeDef(
@@ -156,7 +156,7 @@
     def __init__(self, space):
         self.space = space
         w_module = space.getbuiltinmodule('_continuation')
-        self.w_error = space.getattr(w_module, space.wrap('error'))
+        self.w_error = space.getattr(w_module, space.newtext('error'))
         # the following function switches away immediately, so that
         # continulet.__init__() doesn't immediately run func(), but it
         # also has the hidden purpose of making sure we have a single
@@ -179,7 +179,7 @@
 
 def geterror(space, message):
     cs = space.fromcache(State)
-    return OperationError(cs.w_error, space.wrap(message))
+    return OperationError(cs.w_error, space.newtext(message))
 
 def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
--- a/pypy/module/_continuation/interp_pickle.py
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -24,8 +24,8 @@
     elif self.sthread.is_empty_handle(self.h):
         w_frame = space.w_None
     else:
-        w_frame = space.wrap(self.bottomframe)
-    w_continulet_type = space.type(space.wrap(self))
+        w_frame = self.bottomframe
+    w_continulet_type = space.type(self)
     w_dict = self.getdict(space) or space.w_None
     args = [getunpickle(space),
             space.newtuple([w_continulet_type]),

From pypy.commits at gmail.com  Thu Nov  3 05:50:59 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:50:59 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: pyexpat
Message-ID: <581b0883.010a1c0a.23258.8004@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88102:2304d65e5814
Date: 2016-11-03 08:15 +0100
http://bitbucket.org/pypy/pypy/changeset/2304d65e5814/

Log:	pyexpat

diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -8,7 +8,7 @@
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
         for name in interp_pyexpat.xml_error_list:
-            self.space.setattr(self, self.space.wrap(name),
+            self.space.setattr(self, self.space.newtext(name),
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
@@ -22,7 +22,7 @@
         space = self.space
         for name in interp_pyexpat.xml_model_list:
             value = getattr(interp_pyexpat, name)
-            space.setattr(self, space.wrap(name), space.wrap(value))
+            space.setattr(self, space.newtext(name), space.wrap(value))
 
 class Module(MixedModule):
     "Python wrapper for Expat parser."
@@ -38,7 +38,7 @@
         'ExpatError':    'space.fromcache(interp_pyexpat.Cache).w_error',
         'error':         'space.fromcache(interp_pyexpat.Cache).w_error',
 
-        '__version__':   'space.wrap("85819")',
+        '__version__':   'space.newtext("85819")',
         }
 
     submodules = {
@@ -54,6 +54,6 @@
     def startup(self, space):
         from pypy.module.pyexpat import interp_pyexpat
         w_ver = interp_pyexpat.get_expat_version(space)
-        space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver)
+        space.setattr(self, space.newtext("EXPAT_VERSION"), w_ver)
         w_ver = interp_pyexpat.get_expat_version_info(space)
-        space.setattr(self, space.wrap("version_info"), w_ver)
+        space.setattr(self, space.newtext("version_info"), w_ver)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -487,9 +487,9 @@
     def w_convert(self, space, s):
         if self.returns_unicode:
             from pypy.interpreter.unicodehelper import decode_utf8
-            return space.wrap(decode_utf8(space, s))
+            return space.newunicode(decode_utf8(space, s))
         else:
-            return space.wrap(s)
+            return space.newtext(s)
 
     def w_convert_charp(self, space, data):
         if data:
@@ -591,8 +591,8 @@
         # Yes, supports only 8bit encodings
         translationmap = space.unicode_w(
             space.call_method(
-                space.wrap(self.all_chars), "decode",
-                space.wrap(name), space.wrap("replace")))
+                space.newbytes(self.all_chars), "decode",
+                space.newtext(name), space.newtext("replace")))
 
         if len(translationmap) != 256:
             raise oefmt(space.w_ValueError,
@@ -627,7 +627,7 @@
 
 
     def get_namespace_prefixes(self, space):
-        return space.wrap(self.ns_prefixes)
+        return space.newbool(self.ns_prefixes)
 
     def set_namespace_prefixes(self, space, w_value):
         self.ns_prefixes = space.bool_w(w_value)
@@ -649,14 +649,14 @@
             exc = self.set_error(space, XML_GetErrorCode(self.itself))
             raise exc
         self.flush_character_buffer(space)
-        return space.wrap(res)
+        return space.newint(res)
 
     def ParseFile(self, space, w_file):
         """ParseFile(file)
 Parse XML data from file-like object."""
         eof = False
         while not eof:
-            w_data = space.call_method(w_file, 'read', space.wrap(2048))
+            w_data = space.call_method(w_file, 'read', space.newint(2048))
             data = space.str_w(w_data)
             eof = len(data) == 0
             w_res = self.Parse(space, data, isfinal=eof)
@@ -691,13 +691,13 @@
         for i in range(NB_HANDLERS):
             parser.handlers[i] = self.handlers[i]
 
-        return space.wrap(parser)
+        return parser
 
     def flush_character_buffer(self, space):
         if not self.buffer:
             return
         w_data = space.call_function(
-            space.getattr(space.wrap(''), space.wrap('join')),
+            space.getattr(space.newtext(''), space.newtext('join')),
             space.newlist(self.buffer))
         self.buffer = []
         self.buffer_used = 0
@@ -713,28 +713,28 @@
         colno = XML_GetCurrentColumnNumber(self.itself)
         msg = "%s: line %d, column %d" % (err, lineno, colno)
         w_errorcls = space.fromcache(Cache).w_error
-        w_error = space.call_function(w_errorcls, space.wrap(msg))
-        space.setattr(w_error, space.wrap("code"), space.wrap(code))
-        space.setattr(w_error, space.wrap("offset"), space.wrap(colno))
-        space.setattr(w_error, space.wrap("lineno"), space.wrap(lineno))
+        w_error = space.call_function(w_errorcls, space.newtext(msg))
+        space.setattr(w_error, space.newtext("code"), space.newint(code))
+        space.setattr(w_error, space.newtext("offset"), space.newint(colno))
+        space.setattr(w_error, space.newtext("lineno"), space.newint(lineno))
 
         self.w_error = w_error
         return OperationError(w_errorcls, w_error)
 
     def descr_ErrorCode(self, space):
-        return space.wrap(XML_GetErrorCode(self.itself))
+        return space.newint(XML_GetErrorCode(self.itself))
 
     def descr_ErrorLineNumber(self, space):
-        return space.wrap(XML_GetErrorLineNumber(self.itself))
+        return space.newint(XML_GetErrorLineNumber(self.itself))
 
     def descr_ErrorColumnNumber(self, space):
-        return space.wrap(XML_GetErrorColumnNumber(self.itself))
+        return space.newint(XML_GetErrorColumnNumber(self.itself))
 
     def descr_ErrorByteIndex(self, space):
-        return space.wrap(XML_GetErrorByteIndex(self.itself))
+        return space.newint(XML_GetErrorByteIndex(self.itself))
 
     def get_buffer_size(self, space):
-        return space.wrap(self.buffer_size)
+        return space.newint(self.buffer_size)
     def set_buffer_size(self, space, w_value):
         value = space.getindex_w(w_value, space.w_TypeError)
         if value <= 0:
@@ -744,7 +744,7 @@
         self.buffer_size = value
 
     def get_buffer_text(self, space):
-        return space.wrap(self.buffer is not None)
+        return space.newbool(self.buffer is not None)
     def set_buffer_text(self, space, w_value):
         if space.is_true(w_value):
             self.buffer = []
@@ -762,7 +762,7 @@
 
 def bool_property(name, cls, doc=None):
     def fget(space, obj):
-        return space.wrap(getattr(obj, name))
+        return space.newbool(getattr(obj, name))
     def fset(space, obj, value):
         setattr(obj, name, space.bool_w(value))
     return GetSetProperty(fget, fset, cls=cls, doc=doc)
@@ -859,11 +859,11 @@
     XML_SetUnknownEncodingHandler(
         parser.itself, UnknownEncodingHandlerData_callback,
         rffi.cast(rffi.VOIDP, parser.id))
-    return space.wrap(parser)
+    return parser
 
 @unwrap_spec(code=int)
 def ErrorString(space, code):
     """ErrorString(errno) -> string
 Returns string error for given number."""
-    return space.wrap(rffi.charp2str(XML_ErrorString(code)))
+    return space.newtext(rffi.charp2str(XML_ErrorString(code)))
 

From pypy.commits at gmail.com  Thu Nov  3 05:51:00 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:00 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: faulthandler
Message-ID: <581b0884.4a6ec20a.b8d27.62a7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88103:4560c7ff2bbe
Date: 2016-11-03 08:16 +0100
http://bitbucket.org/pypy/pypy/changeset/4560c7ff2bbe/

Log:	faulthandler

diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -153,7 +153,7 @@
 
 def is_enabled(space):
     "is_enabled()->bool: check if the handler is enabled"
-    return space.wrap(space.fromcache(Handler).is_enabled())
+    return space.newbool(space.fromcache(Handler).is_enabled())
 
 @unwrap_spec(all_threads=int)
 def dump_traceback(space, w_file=None, all_threads=0):
@@ -178,7 +178,7 @@
 
 @unwrap_spec(signum=int)
 def unregister(space, signum):
-    return space.wrap(space.fromcache(Handler).unregister(signum))
+    return space.newbool(space.fromcache(Handler).unregister(signum))
 
 
 # for tests...
@@ -206,4 +206,4 @@
 @unwrap_spec(levels=int)
 def stack_overflow(space, levels=100000000):
     levels = float(levels)
-    return space.wrap(cintf.pypy_faulthandler_stackoverflow(levels))
+    return space.newfloat(cintf.pypy_faulthandler_stackoverflow(levels))

From pypy.commits at gmail.com  Thu Nov  3 05:50:55 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:50:55 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: posix
Message-ID: <581b087f.13d41c0a.b11b.7ffe@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88100:b3e96486e8ae
Date: 2016-11-03 07:59 +0100
http://bitbucket.org/pypy/pypy/changeset/b3e96486e8ae/

Log:	posix

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -31,9 +31,9 @@
 
 def wrap_uid(space, uid):
     if uid <= r_uint(sys.maxint):
-        return space.wrap(intmask(uid))
+        return space.newint(intmask(uid))
     else:
-        return space.wrap(uid)     # an unsigned number
+        return space.newint(uid)     # an unsigned number
 wrap_gid = wrap_uid
 
 def fsencode_w(space, w_obj):
@@ -112,7 +112,7 @@
             space, w_fname, flag, mode)
     except OSError as e:
         raise wrap_oserror2(space, e, w_fname)
-    return space.wrap(fd)
+    return space.newint(fd)
 
 @unwrap_spec(fd=c_int, pos=r_longlong, how=c_int)
 def lseek(space, fd, pos, how):
@@ -124,7 +124,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(pos)
+        return space.newint(pos)
 
 @unwrap_spec(fd=c_int)
 def isatty(space, fd):
@@ -135,7 +135,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(res)
+        return space.newbool(res)
 
 @unwrap_spec(fd=c_int, buffersize=int)
 def read(space, fd, buffersize):
@@ -145,7 +145,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(s)
+        return space.newbytes(s)
 
 @unwrap_spec(fd=c_int)
 def write(space, fd, w_data):
@@ -157,7 +157,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(res)
+        return space.newint(res)
 
 @unwrap_spec(fd=c_int)
 def close(space, fd):
@@ -180,9 +180,9 @@
     except IOError as e:
         if not objectmodel.we_are_translated():
             # Python 2.6 raises an IOError here. Let's not repeat that mistake.
-            w_error = space.call_function(space.w_OSError, space.wrap(e.errno),
-                                          space.wrap(e.strerror),
-                                          space.wrap(e.filename))
+            w_error = space.call_function(space.w_OSError, space.newint(e.errno),
+                                          space.newtext(e.strerror),
+                                          space.newtext(e.filename))
             raise OperationError(space.w_OSError, w_error)
         raise AssertionError
     except OSError as e:
@@ -229,35 +229,35 @@
         if i < rposix_stat.N_INDEXABLE_FIELDS:
             # get the first 10 items by indexing; this gives us
             # 'st_Xtime' as an integer, too
-            w_value = space.wrap(st[i])
+            w_value = space.newint(st[i])
             lst[i] = w_value
         elif name.startswith('st_'):    # exclude 'nsec_Xtime'
-            w_value = space.wrap(getattr(st, name))
-            space.setitem(w_keywords, space.wrap(name), w_value)
+            w_value = space.newint(getattr(st, name))
+            space.setitem(w_keywords, space.newtext(name), w_value)
 
     # non-rounded values for name-based access
     if stat_float_times:
         space.setitem(w_keywords,
-                      space.wrap('st_atime'), space.wrap(st.st_atime))
+                      space.newtext('st_atime'), space.newfloat(st.st_atime))
         space.setitem(w_keywords,
-                      space.wrap('st_mtime'), space.wrap(st.st_mtime))
+                      space.newtext('st_mtime'), space.newfloat(st.st_mtime))
         space.setitem(w_keywords,
-                      space.wrap('st_ctime'), space.wrap(st.st_ctime))
+                      space.newtext('st_ctime'), space.newfloat(st.st_ctime))
     #else:
     #   filled by the __init__ method
 
     w_tuple = space.newtuple(lst)
     w_stat_result = space.getattr(space.getbuiltinmodule(os.name),
-                                  space.wrap('stat_result'))
+                                  space.newtext('stat_result'))
     return space.call_function(w_stat_result, w_tuple, w_keywords)
 
 
 def build_statvfs_result(space, st):
     vals_w = [None] * len(rposix_stat.STATVFS_FIELDS)
     for i, (name, _) in STATVFS_FIELDS:
-        vals_w[i] = space.wrap(getattr(st, name))
+        vals_w[i] = space.newint(getattr(st, name))
     w_tuple = space.newtuple(vals_w)
-    w_statvfs_result = space.getattr(space.getbuiltinmodule(os.name), space.wrap('statvfs_result'))
+    w_statvfs_result = space.getattr(space.getbuiltinmodule(os.name), space.newtext('statvfs_result'))
     return space.call_function(w_statvfs_result, w_tuple)
 
 
@@ -318,7 +318,7 @@
     state = space.fromcache(StatState)
 
     if w_value is None:
-        return space.wrap(state.stat_float_times)
+        return space.newbool(state.stat_float_times)
     else:
         state.stat_float_times = space.bool_w(w_value)
 
@@ -351,7 +351,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(newfd)
+        return space.newint(newfd)
 
 @unwrap_spec(old_fd=c_int, new_fd=c_int)
 def dup2(space, old_fd, new_fd):
@@ -377,7 +377,7 @@
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
-        return space.wrap(ok)
+        return space.newint(ok)
 
 
 def times(space):
@@ -391,11 +391,11 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.newtuple([space.wrap(times[0]),
-                               space.wrap(times[1]),
-                               space.wrap(times[2]),
-                               space.wrap(times[3]),
-                               space.wrap(times[4])])
+        return space.newtuple([space.newfloat(times[0]),
+                               space.newfloat(times[1]),
+                               space.newfloat(times[2]),
+                               space.newfloat(times[3]),
+                               space.newfloat(times[4])])
 
 @unwrap_spec(cmd='str0')
 def system(space, cmd):
@@ -405,7 +405,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(rc)
+        return space.newint(rc)
 
 def unlink(space, w_path):
     """Remove a file (same as remove(path))."""
@@ -427,11 +427,11 @@
         if space.isinstance_w(w_path, space.w_unicode):
             path = FileEncoder(space, w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.wrap(fullpath)
+            w_fullpath = space.newtext(fullpath)
         else:
             path = space.str0_w(w_path)
             fullpath = rposix.getfullpathname(path)
-            w_fullpath = space.wrap(fullpath)
+            w_fullpath = space.newtext(fullpath)
     except OSError as e:
         raise wrap_oserror2(space, e, w_path)
     else:
@@ -444,7 +444,7 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(cur)
+        return space.newtext(cur)
 
 if _WIN32:
     def getcwdu(space):
@@ -454,7 +454,7 @@
         except OSError as e:
             raise wrap_oserror(space, e)
         else:
-            return space.wrap(cur)
+            return space.newunicode(cur)
 else:
     def getcwdu(space):
         """Return the current working directory as a unicode string."""
@@ -491,7 +491,7 @@
         text = os.strerror(errno)
     except ValueError:
         raise oefmt(space.w_ValueError, "strerror() argument out of range")
-    return space.wrap(text)
+    return space.newtext(text)
 
 def getlogin(space):
     """Return the currently logged in user."""
@@ -500,14 +500,14 @@
     except OSError as e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(cur)
+        return space.newtext(cur)
 
 # ____________________________________________________________
 
 def getstatfields(space):
     # for app_posix.py: export the list of 'st_xxx' names that we know
     # about at RPython level
-    return space.newlist([space.wrap(name) for _, (name, _) in STAT_FIELDS])
+    return space.newlist([space.newtext(name) for _, (name, _) in STAT_FIELDS])
 
 
 class State:
@@ -531,7 +531,7 @@
 def _convertenviron(space, w_env):
     space.call_method(w_env, 'clear')
     for key, value in os.environ.items():
-        space.setitem(w_env, space.wrap(key), space.wrap(value))
+        space.setitem(w_env, space.newtext(key), space.newtext(value))
 
 @unwrap_spec(name='str0', value='str0')
 def putenv(space, name, value):
@@ -579,7 +579,7 @@
             len_result = len(result)
             result_w = [None] * len_result
             for i in range(len_result):
-                w_bytes = space.wrap(result[i])
+                w_bytes = space.newtext(result[i])
                 try:
                     result_w[i] = space.call_method(w_bytes,
                                                     "decode", w_fs_encoding)
@@ -604,7 +604,7 @@
         fd1, fd2 = os.pipe()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(fd1), space.wrap(fd2)])
+    return space.newtuple([space.newint(fd1), space.newint(fd2)])
 
 @unwrap_spec(mode=c_int)
 def chmod(space, w_path, mode):
@@ -656,7 +656,7 @@
 def umask(space, mask):
     "Set the current numeric umask and return the previous umask."
     prevmask = os.umask(mask)
-    return space.wrap(prevmask)
+    return space.newint(prevmask)
 
 def getpid(space):
     "Return the current process id."
@@ -664,7 +664,7 @@
         pid = os.getpid()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(pid)
+    return space.newint(pid)
 
 @unwrap_spec(pid=c_int, sig=c_int)
 def kill(space, pid, sig):
@@ -710,7 +710,7 @@
         result = os.readlink(path)
     except OSError as e:
         raise wrap_oserror(space, e, path)
-    return space.wrap(result)
+    return space.newtext(result)
 
 before_fork_hooks = []
 after_fork_child_hooks = []
@@ -763,7 +763,7 @@
 
 def fork(space):
     pid, irrelevant = _run_forking_function(space, "F")
-    return space.wrap(pid)
+    return space.newint(pid)
 
 def openpty(space):
     "Open a pseudo-terminal, returning open fd's for both master and slave end."
@@ -771,12 +771,12 @@
         master_fd, slave_fd = os.openpty()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)])
+    return space.newtuple([space.newint(master_fd), space.newint(slave_fd)])
 
 def forkpty(space):
     pid, master_fd = _run_forking_function(space, "P")
-    return space.newtuple([space.wrap(pid),
-                           space.wrap(master_fd)])
+    return space.newtuple([space.newint(pid),
+                           space.newint(master_fd)])
 
 @unwrap_spec(pid=c_int, options=c_int)
 def waitpid(space, pid, options):
@@ -788,7 +788,7 @@
         pid, status = os.waitpid(pid, options)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.newtuple([space.wrap(pid), space.wrap(status)])
+    return space.newtuple([space.newint(pid), space.newint(status)])
 
 @unwrap_spec(status=c_int)
 def _exit(space, status):
@@ -853,7 +853,7 @@
         ret = os.spawnv(mode, path, args)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(ret)
+    return space.newint(ret)
 
 @unwrap_spec(mode=int, path='str0')
 def spawnve(space, mode, path, w_args, w_env):
@@ -863,7 +863,7 @@
         ret = os.spawnve(mode, path, args, env)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(ret)
+    return space.newint(ret)
 
 def utime(space, w_path, w_tuple):
     """ utime(path, (atime, mtime))
@@ -902,7 +902,7 @@
         r = os.uname()
     except OSError as e:
         raise wrap_oserror(space, e)
-    l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]]
+    l_w = [space.newtext(i) for i in [r[0], r[1], r[2], r[3], r[4]]]
     return space.newtuple(l_w)
 
 def getuid(space):
@@ -1031,7 +1031,7 @@
 
     Return the current process group id.
     """
-    return space.wrap(os.getpgrp())
+    return space.newint(os.getpgrp())
 
 def setpgrp(space):
     """ setpgrp()
@@ -1049,7 +1049,7 @@
 
     Return the parent's process id.
     """
-    return space.wrap(os.getppid())
+    return space.newint(os.getppid())
 
 @unwrap_spec(pid=c_int)
 def getpgid(space, pid):
@@ -1061,7 +1061,7 @@
         pgid = os.getpgid(pid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(pgid)
+    return space.newint(pgid)
 
 @unwrap_spec(pid=c_int, pgrp=c_int)
 def setpgid(space, pid, pgrp):
@@ -1107,7 +1107,7 @@
         sid = os.getsid(pid)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(sid)
+    return space.newint(sid)
 
 def setsid(space):
     """ setsid()
@@ -1130,7 +1130,7 @@
         pgid = os.tcgetpgrp(fd)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(pgid)
+    return space.newint(pgid)
 
 @unwrap_spec(fd=c_int, pgid=c_int)
 def tcsetpgrp(space, fd, pgid):
@@ -1195,7 +1195,7 @@
     if name in ('WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG'):
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
-            return space.wrap(getattr(os, name)(status))
+            return space.newint(getattr(os, name)(status))
     else:
         @unwrap_spec(status=c_int)
         def WSTAR(space, status):
@@ -1235,7 +1235,7 @@
         res = os.sysconf(num)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(res)
+    return space.newint(res)
 
 @unwrap_spec(fd=c_int)
 def fpathconf(space, fd, w_name):
@@ -1244,7 +1244,7 @@
         res = os.fpathconf(fd, num)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(res)
+    return space.newint(res)
 
 @unwrap_spec(path='str0')
 def pathconf(space, path, w_name):
@@ -1253,7 +1253,7 @@
         res = os.pathconf(path, num)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(res)
+    return space.newint(res)
 
 def confstr(space, w_name):
     num = confname_w(space, w_name, os.confstr_names)
@@ -1261,7 +1261,7 @@
         res = os.confstr(num)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(res)
+    return space.newtext(res)
 
 @unwrap_spec(path='str0', uid=c_uid_t, gid=c_gid_t)
 def chown(space, path, uid, gid):
@@ -1295,24 +1295,24 @@
         load = os.getloadavg()
     except OSError:
         raise oefmt(space.w_OSError, "Load averages are unobtainable")
-    return space.newtuple([space.wrap(load[0]),
-                           space.wrap(load[1]),
-                           space.wrap(load[2])])
+    return space.newtuple([space.newfloat(load[0]),
+                           space.newfloat(load[1]),
+                           space.newfloat(load[2])])
 
 @unwrap_spec(major=c_int, minor=c_int)
 def makedev(space, major, minor):
     result = os.makedev(major, minor)
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def major(space, device):
     result = os.major(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(device="c_uint")
 def minor(space, device):
     result = os.minor(intmask(device))
-    return space.wrap(result)
+    return space.newint(result)
 
 @unwrap_spec(inc=c_int)
 def nice(space, inc):
@@ -1321,7 +1321,7 @@
         res = os.nice(inc)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(res)
+    return space.newint(res)
 
 @unwrap_spec(n=int)
 def urandom(space, n):
@@ -1331,7 +1331,7 @@
     """
     context = get(space).random_context
     try:
-        return space.wrap(rurandom.urandom(context, n))
+        return space.newbytes(rurandom.urandom(context, n))
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -1340,4 +1340,4 @@
 
     Return the name of the controlling terminal for this process.
     """
-    return space.wrap(os.ctermid())
+    return space.newtext(os.ctermid())

From pypy.commits at gmail.com  Thu Nov  3 05:51:02 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:02 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: fix small longs
Message-ID: <581b0886.e271c20a.57e46.6349@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88104:7e4212bebd66
Date: 2016-11-03 08:30 +0100
http://bitbucket.org/pypy/pypy/changeset/7e4212bebd66/

Log:	fix small longs

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -242,7 +242,7 @@
 
     @specialize.argtype(1)
     def newint(self, intval):
-        if self.config.objspace.std.withsmalllong:
+        if self.config.objspace.std.withsmalllong and isinstance(intval, base_int):
             from pypy.objspace.std.smalllongobject import W_SmallLongObject
             from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
             from rpython.rlib.rarithmetic import longlongmax

From pypy.commits at gmail.com  Thu Nov  3 05:51:04 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:04 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: marshal
Message-ID: <581b0888.a8c8c20a.b63e.6676@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88105:b4aef31f1f15
Date: 2016-11-03 08:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b4aef31f1f15/

Log:	marshal

diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -31,7 +31,7 @@
 by dump(data, file)."""
     m = StringMarshaller(space, space.int_w(w_version))
     m.dump_w_obj(w_data)
-    return space.wrap(m.get_value())
+    return space.newbytes(m.get_value())
 
 def load(space, w_f):
     """Read one value from the file 'f' and return it."""
@@ -75,7 +75,7 @@
     def __init__(self, space, w_f):
         AbstractReaderWriter.__init__(self, space)
         try:
-            self.func = space.getattr(w_f, space.wrap('write'))
+            self.func = space.getattr(w_f, space.newtext('write'))
             # XXX how to check if it is callable?
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
@@ -85,14 +85,14 @@
 
     def write(self, data):
         space = self.space
-        space.call_function(self.func, space.wrap(data))
+        space.call_function(self.func, space.newbytes(data))
 
 
 class FileReader(AbstractReaderWriter):
     def __init__(self, space, w_f):
         AbstractReaderWriter.__init__(self, space)
         try:
-            self.func = space.getattr(w_f, space.wrap('read'))
+            self.func = space.getattr(w_f, space.newtext('read'))
             # XXX how to check if it is callable?
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
@@ -102,7 +102,7 @@
 
     def read(self, n):
         space = self.space
-        w_ret = space.call_function(self.func, space.wrap(n))
+        w_ret = space.call_function(self.func, space.newint(n))
         ret = space.str_w(w_ret)
         if len(ret) != n:
             self.raise_eof()
@@ -133,7 +133,7 @@
 class _Base(object):
     def raise_exc(self, msg):
         space = self.space
-        raise OperationError(space.w_ValueError, space.wrap(msg))
+        raise OperationError(space.w_ValueError, space.newtext(msg))
 
 class Marshaller(_Base):
     """

From pypy.commits at gmail.com  Thu Nov  3 05:51:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:05 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: time
Message-ID: <581b0889.cf9a1c0a.cc318.f498@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88106:5fc502c79448
Date: 2016-11-03 08:37 +0100
http://bitbucket.org/pypy/pypy/changeset/5fc502c79448/

Log:	time

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -209,7 +209,7 @@
         accept2dyear = 0
     else:
         accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.wrap(accept2dyear))
+    _set_module_object(space, "accept2dyear", space.newtext(accept2dyear))
 
 def _init_timezone(space):
     timezone = daylight = altzone = 0
@@ -351,12 +351,12 @@
 
 def _get_module_object(space, obj_name):
     w_module = space.getbuiltinmodule('time')
-    w_obj = space.getattr(w_module, space.wrap(obj_name))
+    w_obj = space.getattr(w_module, space.newtext(obj_name))
     return w_obj
 
 def _set_module_object(space, obj_name, w_obj_value):
     w_module = space.getbuiltinmodule('time')
-    space.setattr(w_module, space.wrap(obj_name), w_obj_value)
+    space.setattr(w_module, space.newtext(obj_name), w_obj_value)
 
 def _get_inttime(space, w_seconds):
     # w_seconds can be a wrapped None (it will be automatically wrapped
@@ -379,15 +379,15 @@
 
 def _tm_to_tuple(space, t):
     time_tuple = [
-        space.wrap(rffi.getintfield(t, 'c_tm_year') + 1900),
-        space.wrap(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_mday')),
-        space.wrap(rffi.getintfield(t, 'c_tm_hour')),
-        space.wrap(rffi.getintfield(t, 'c_tm_min')),
-        space.wrap(rffi.getintfield(t, 'c_tm_sec')),
-        space.wrap((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
-        space.wrap(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
-        space.wrap(rffi.getintfield(t, 'c_tm_isdst'))]
+        space.newint(rffi.getintfield(t, 'c_tm_year') + 1900),
+        space.newint(rffi.getintfield(t, 'c_tm_mon') + 1), # want january == 1
+        space.newint(rffi.getintfield(t, 'c_tm_mday')),
+        space.newint(rffi.getintfield(t, 'c_tm_hour')),
+        space.newint(rffi.getintfield(t, 'c_tm_min')),
+        space.newint(rffi.getintfield(t, 'c_tm_sec')),
+        space.newint((rffi.getintfield(t, 'c_tm_wday') + 6) % 7), # want monday == 0
+        space.newint(rffi.getintfield(t, 'c_tm_yday') + 1), # want january, 1 == 1
+        space.newint(rffi.getintfield(t, 'c_tm_isdst'))]
 
     w_struct_time = _get_module_object(space, 'struct_time')
     w_time_tuple = space.newtuple(time_tuple)
@@ -405,7 +405,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                space.wrap(_get_error_msg()))
+                space.newtext(_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)
@@ -476,7 +476,7 @@
     Fractions of a second may be present if the system clock provides them."""
 
     secs = pytime.time()
-    return space.wrap(secs)
+    return space.newfloat(secs)
 
 def clock(space):
     """clock() -> floating point number
@@ -485,7 +485,7 @@
     the first call to clock().  This has as much precision as the system
     records."""
 
-    return space.wrap(pytime.clock())
+    return space.newfloat(pytime.clock())
 
 def ctime(space, w_seconds=None):
     """ctime([seconds]) -> string
@@ -503,7 +503,7 @@
     if not p:
         raise oefmt(space.w_ValueError, "unconvertible time")
 
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
+    return space.newtext(rffi.charp2str(p)[:-1]) # get rid of new line
 
 # by now w_tup is an optional argument (and not *args)
 # because of the ext. compiler bugs in handling such arguments (*args, **kwds)
@@ -518,7 +518,7 @@
     if not p:
         raise oefmt(space.w_ValueError, "unconvertible time")
 
-    return space.wrap(rffi.charp2str(p)[:-1]) # get rid of new line
+    return space.newtext(rffi.charp2str(p)[:-1]) # get rid of new line
 
 def gmtime(space, w_seconds=None):
     """gmtime([seconds]) -> (tm_year, tm_mon, tm_day, tm_hour, tm_min,
@@ -537,7 +537,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_ValueError, space.newtext(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def localtime(space, w_seconds=None):
@@ -554,7 +554,7 @@
     lltype.free(t_ref, flavor='raw')
 
     if not p:
-        raise OperationError(space.w_ValueError, space.wrap(_get_error_msg()))
+        raise OperationError(space.w_ValueError, space.newtext(_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
@@ -570,7 +570,7 @@
     if tt == -1 and rffi.getintfield(buf, "c_tm_wday") == -1:
         raise oefmt(space.w_OverflowError, "mktime argument out of range")
 
-    return space.wrap(float(tt))
+    return space.newfloat(float(tt))
 
 if _POSIX:
     def tzset(space):
@@ -645,7 +645,7 @@
                 # e.g. an empty format, or %Z when the timezone
                 # is unknown.
                 result = rffi.charp2strn(outbuf, intmask(buflen))
-                return space.wrap(result)
+                return space.newtext(result)
         finally:
             lltype.free(outbuf, flavor='raw')
         i += i

From pypy.commits at gmail.com  Thu Nov  3 05:51:07 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:07 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: crypt
Message-ID: <581b088b.832ec20a.364a8.65c0@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88107:de24352afe1b
Date: 2016-11-03 08:37 +0100
http://bitbucket.org/pypy/pypy/changeset/de24352afe1b/

Log:	crypt

diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -21,4 +21,4 @@
     if not res:
         return space.w_None
     str_res = rffi.charp2str(res)
-    return space.wrap(str_res)
+    return space.newtext(str_res)

From pypy.commits at gmail.com  Thu Nov  3 05:51:09 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:09 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: various
Message-ID: <581b088d.d5901c0a.4372d.82be@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88108:641b37bc9cd5
Date: 2016-11-03 08:37 +0100
http://bitbucket.org/pypy/pypy/changeset/641b37bc9cd5/

Log:	various

diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -15,7 +15,7 @@
             errors = 'strict'
         self.space = space
         self.errors = errors
-        w_codec = space.getattr(space.wrap(self), space.wrap("codec"))
+        w_codec = space.getattr(self, space.newtext("codec"))
         codec = space.interp_w(MultibyteCodec, w_codec)
         self.codec = codec.codec
         self.name = codec.name
@@ -30,7 +30,7 @@
         self._initialize()
 
     def fget_errors(self, space):
-        return space.wrap(self.errors)
+        return space.newtext(self.errors)
 
     def fset_errors(self, space, w_errors):
         self.errors = space.str_w(w_errors)
@@ -65,14 +65,14 @@
         pos = c_codecs.pypy_cjk_dec_inbuf_consumed(self.decodebuf)
         assert 0 <= pos <= len(object)
         self.pending = object[pos:]
-        return space.wrap(output)
+        return space.newunicode(output)
 
 
 @unwrap_spec(errors="str_or_None")
 def mbidecoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalDecoder, w_subtype)
     r.__init__(space, errors)
-    return space.wrap(r)
+    return r
 
 MultibyteIncrementalDecoder.typedef = TypeDef(
     'MultibyteIncrementalDecoder',
@@ -120,7 +120,7 @@
 def mbiencoder_new(space, w_subtype, errors=None):
     r = space.allocate_instance(MultibyteIncrementalEncoder, w_subtype)
     r.__init__(space, errors)
-    return space.wrap(r)
+    return r
 
 MultibyteIncrementalEncoder.typedef = TypeDef(
     'MultibyteIncrementalEncoder',
diff --git a/pypy/module/marshal/__init__.py b/pypy/module/marshal/__init__.py
--- a/pypy/module/marshal/__init__.py
+++ b/pypy/module/marshal/__init__.py
@@ -14,5 +14,5 @@
         'dumps'   : 'interp_marshal.dumps',
         'load'    : 'interp_marshal.load',
         'loads'   : 'interp_marshal.loads',
-        'version' : 'space.wrap(interp_marshal.Py_MARSHAL_VERSION)',
+        'version' : 'space.newint(interp_marshal.Py_MARSHAL_VERSION)',
     }
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1213,7 +1213,7 @@
 @unwrap_spec(fd=c_int)
 def ttyname(space, fd):
     try:
-        return space.wrap(os.ttyname(fd))
+        return space.newtext(os.ttyname(fd))
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -209,7 +209,7 @@
         accept2dyear = 0
     else:
         accept2dyear = 1
-    _set_module_object(space, "accept2dyear", space.newtext(accept2dyear))
+    _set_module_object(space, "accept2dyear", space.newint(accept2dyear))
 
 def _init_timezone(space):
     timezone = daylight = altzone = 0

From pypy.commits at gmail.com  Thu Nov  3 05:51:10 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:10 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: remove the simple wraps from
 cpyext (left more complex cases for now)
Message-ID: <581b088e.28c9c20a.cb9c0.6700@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88109:fdd451dbedab
Date: 2016-11-03 10:31 +0100
http://bitbucket.org/pypy/pypy/changeset/fdd451dbedab/

Log:	remove the simple wraps from cpyext (left more complex cases for
	now)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -911,7 +911,7 @@
                 else:
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
-                                                   space.wrap(message)))
+                                                   space.newtext(message)))
             else:
                 failed = False
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -305,9 +305,9 @@
 
     w_encoding = w_errors = None
     if encoding:
-        w_encoding = space.wrap(rffi.charp2str(encoding))
+        w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
 @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject)
@@ -325,9 +325,9 @@
 
     w_encoding = w_errors = None
     if encoding:
-        w_encoding = space.wrap(rffi.charp2str(encoding))
+        w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
     return space.call_method(w_str, "decode", w_encoding, w_errors)
 
 @cpython_api([PyObject, PyObject], PyObject)
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -41,25 +41,25 @@
     datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw',
                                 track_allocation=False)
 
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
 
-    w_type = space.getattr(w_datetime, space.wrap("date"))
+    w_type = space.getattr(w_datetime, space.newtext("date"))
     datetimeAPI.c_DateType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("datetime"))
     datetimeAPI.c_DateTimeType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("time"))
+    w_type = space.getattr(w_datetime, space.newtext("time"))
     datetimeAPI.c_TimeType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("timedelta"))
+    w_type = space.getattr(w_datetime, space.newtext("timedelta"))
     datetimeAPI.c_DeltaType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
-    w_type = space.getattr(w_datetime, space.wrap("tzinfo"))
+    w_type = space.getattr(w_datetime, space.newtext("tzinfo"))
     datetimeAPI.c_TZInfoType = rffi.cast(
         PyTypeObjectPtr, make_ref(space, w_type))
 
@@ -137,7 +137,7 @@
     day = rffi.cast(lltype.Signed, day)
     return space.call_function(
         w_type,
-        space.wrap(year), space.wrap(month), space.wrap(day))
+        space.newint(year), space.newint(month), space.newint(day))
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
               PyObject, PyTypeObjectPtr], PyObject)
@@ -150,8 +150,8 @@
     usecond = rffi.cast(lltype.Signed, usecond)
     return space.call_function(
         w_type,
-        space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond), w_tzinfo)
+        space.newint(hour), space.newint(minute), space.newint(second),
+        space.newint(usecond), w_tzinfo)
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real,
               rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
@@ -171,18 +171,18 @@
     usecond = rffi.cast(lltype.Signed, usecond)
     return space.call_function(
         w_type,
-        space.wrap(year), space.wrap(month), space.wrap(day),
-        space.wrap(hour), space.wrap(minute), space.wrap(second),
-        space.wrap(usecond), w_tzinfo)
+        space.newint(year), space.newint(month), space.newint(day),
+        space.newint(hour), space.newint(minute), space.newint(second),
+        space.newint(usecond), w_tzinfo)
 
 @cpython_api([PyObject], PyObject)
 def PyDateTime_FromTimestamp(space, w_args):
     """Create and return a new datetime.datetime object given an argument tuple
     suitable for passing to datetime.datetime.fromtimestamp().
     """
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    w_type = space.getattr(w_datetime, space.wrap("datetime"))
-    w_method = space.getattr(w_type, space.wrap("fromtimestamp"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("datetime"))
+    w_method = space.getattr(w_type, space.newtext("fromtimestamp"))
     return space.call(w_method, w_args)
 
 @cpython_api([PyObject], PyObject)
@@ -190,9 +190,9 @@
     """Create and return a new datetime.date object given an argument tuple
     suitable for passing to datetime.date.fromtimestamp().
     """
-    w_datetime = PyImport_Import(space, space.wrap("datetime"))
-    w_type = space.getattr(w_datetime, space.wrap("date"))
-    w_method = space.getattr(w_type, space.wrap("fromtimestamp"))
+    w_datetime = PyImport_Import(space, space.newtext("datetime"))
+    w_type = space.getattr(w_datetime, space.newtext("date"))
+    w_method = space.getattr(w_type, space.newtext("fromtimestamp"))
     return space.call(w_method, w_args)
 
 @cpython_api([rffi.INT_real, rffi.INT_real, rffi.INT_real, rffi.INT_real,
@@ -209,7 +209,7 @@
     useconds = rffi.cast(lltype.Signed, useconds)
     return space.call_function(
         w_type,
-        space.wrap(days), space.wrap(seconds), space.wrap(useconds))
+        space.newint(days), space.newint(seconds), space.newint(useconds))
 
 # Accessors
 
@@ -217,67 +217,67 @@
 def PyDateTime_GET_YEAR(space, w_obj):
     """Return the year, as a positive int.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("year")))
+    return space.int_w(space.getattr(w_obj, space.newtext("year")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_MONTH(space, w_obj):
     """Return the month, as an int from 1 through 12.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("month")))
+    return space.int_w(space.getattr(w_obj, space.newtext("month")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_GET_DAY(space, w_obj):
     """Return the day, as an int from 1 through 31.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("day")))
+    return space.int_w(space.getattr(w_obj, space.newtext("day")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("hour")))
+    return space.int_w(space.getattr(w_obj, space.newtext("hour")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("minute")))
+    return space.int_w(space.getattr(w_obj, space.newtext("minute")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("second")))
+    return space.int_w(space.getattr(w_obj, space.newtext("second")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DATE_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("microsecond")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microsecond")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_HOUR(space, w_obj):
     """Return the hour, as an int from 0 through 23.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("hour")))
+    return space.int_w(space.getattr(w_obj, space.newtext("hour")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MINUTE(space, w_obj):
     """Return the minute, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("minute")))
+    return space.int_w(space.getattr(w_obj, space.newtext("minute")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_SECOND(space, w_obj):
     """Return the second, as an int from 0 through 59.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("second")))
+    return space.int_w(space.getattr(w_obj, space.newtext("second")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_TIME_GET_MICROSECOND(space, w_obj):
     """Return the microsecond, as an int from 0 through 999999.
     """
-    return space.int_w(space.getattr(w_obj, space.wrap("microsecond")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microsecond")))
 
 # XXX these functions are not present in the Python API
 # But it does not seem possible to expose a different structure
@@ -285,12 +285,12 @@
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_DAYS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("days")))
+    return space.int_w(space.getattr(w_obj, space.newtext("days")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_SECONDS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("seconds")))
+    return space.int_w(space.getattr(w_obj, space.newtext("seconds")))
 
 @cpython_api([rffi.VOIDP], rffi.INT_real, error=CANNOT_FAIL)
 def PyDateTime_DELTA_GET_MICROSECONDS(space, w_obj):
-    return space.int_w(space.getattr(w_obj, space.wrap("microseconds")))
+    return space.int_w(space.getattr(w_obj, space.newtext("microseconds")))
diff --git a/pypy/module/cpyext/codecs.py b/pypy/module/cpyext/codecs.py
--- a/pypy/module/cpyext/codecs.py
+++ b/pypy/module/cpyext/codecs.py
@@ -6,7 +6,7 @@
 def PyCodec_IncrementalEncoder(space, encoding, errors):
     w_codec = interp_codecs.lookup_codec(space, rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
         return space.call_method(w_codec, "incrementalencoder", w_errors)
     else:
         return space.call_method(w_codec, "incrementalencoder")
@@ -15,7 +15,7 @@
 def PyCodec_IncrementalDecoder(space, encoding, errors):
     w_codec = interp_codecs.lookup_codec(space, rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
         return space.call_method(w_codec, "incrementaldecoder", w_errors)
     else:
         return space.call_method(w_codec, "incrementaldecoder")
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -74,7 +74,7 @@
         key = rffi.charp2str(key_ptr)
         # our dicts dont have a standardized interface, so we need
         # to go through the space
-        space.delitem(w_dict, space.wrap(key))
+        space.delitem(w_dict, space.newtext(key))
         return 0
     else:
         PyErr_BadInternalCall(space)
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -31,7 +31,7 @@
     caller = space.getexecutioncontext().gettopframe_nohidden()
     if caller is not None:
         w_globals = caller.get_w_globals()
-        w_builtins = space.getitem(w_globals, space.wrap('__builtins__'))
+        w_builtins = space.getitem(w_globals, space.newtext('__builtins__'))
         if not space.isinstance_w(w_builtins, space.w_dict):
             w_builtins = w_builtins.getdict(space)
     else:
diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py
--- a/pypy/module/cpyext/floatobject.py
+++ b/pypy/module/cpyext/floatobject.py
@@ -42,7 +42,7 @@
 
 @cpython_api([lltype.Float], PyObject)
 def PyFloat_FromDouble(space, value):
-    return space.wrap(value)
+    return space.newfloat(value)
 
 @cpython_api([PyObject], lltype.Float, error=-1)
 def PyFloat_AsDouble(space, w_obj):
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -63,9 +63,8 @@
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
     d = frame.getorcreatedebug()
     d.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
-    w_obj = space.wrap(frame)
-    track_reference(space, py_obj, w_obj)
-    return w_obj
+    track_reference(space, py_obj, frame)
+    return frame
 
 @cpython_api([PyThreadState, PyCodeObject, PyObject, PyObject], PyFrameObject,
              result_is_ll=True)
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -54,7 +54,7 @@
 def function_attach(space, py_obj, w_obj):
     py_func = rffi.cast(PyFunctionObject, py_obj)
     assert isinstance(w_obj, Function)
-    py_func.c_func_name = make_ref(space, space.wrap(w_obj.name))
+    py_func.c_func_name = make_ref(space, space.newtext(w_obj.name))
 
 @cpython_api([PyObject], lltype.Void, header=None)
 def function_dealloc(space, py_obj):
@@ -66,8 +66,8 @@
 def code_attach(space, py_obj, w_obj):
     py_code = rffi.cast(PyCodeObject, py_obj)
     assert isinstance(w_obj, PyCode)
-    py_code.c_co_name = make_ref(space, space.wrap(w_obj.co_name))
-    py_code.c_co_filename = make_ref(space, space.wrap(w_obj.co_filename))
+    py_code.c_co_name = make_ref(space, space.newtext(w_obj.co_name))
+    py_code.c_co_filename = make_ref(space, space.newtext(w_obj.co_filename))
     co_flags = 0
     for name, value in ALL_CODE_FLAGS:
         if w_obj.co_flags & getattr(pycode, name):
@@ -87,8 +87,7 @@
 def PyFunction_GetCode(space, w_func):
     """Return the code object associated with the function object op."""
     func = space.interp_w(Function, w_func)
-    w_code = space.wrap(func.code)
-    return w_code      # borrowed ref
+    return func.code      # borrowed ref
 
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PyMethod_New(space, w_func, w_self, w_cls):
@@ -132,40 +131,40 @@
     create a frame, use PyCode_NewEmpty() instead.  Calling
     PyCode_New() directly can bind you to a precise Python
     version since the definition of the bytecode changes often."""
-    return space.wrap(PyCode(space,
-                             argcount=rffi.cast(lltype.Signed, argcount),
-                             nlocals=rffi.cast(lltype.Signed, nlocals),
-                             stacksize=rffi.cast(lltype.Signed, stacksize),
-                             flags=rffi.cast(lltype.Signed, flags),
-                             code=space.str_w(w_code),
-                             consts=space.fixedview(w_consts),
-                             names=unwrap_list_of_strings(space, w_names),
-                             varnames=unwrap_list_of_strings(space, w_varnames),
-                             filename=space.str_w(w_filename),
-                             name=space.str_w(w_funcname),
-                             firstlineno=rffi.cast(lltype.Signed, firstlineno),
-                             lnotab=space.str_w(w_lnotab),
-                             freevars=unwrap_list_of_strings(space, w_freevars),
-                             cellvars=unwrap_list_of_strings(space, w_cellvars)))
+    return PyCode(space,
+                  argcount=rffi.cast(lltype.Signed, argcount),
+                  nlocals=rffi.cast(lltype.Signed, nlocals),
+                  stacksize=rffi.cast(lltype.Signed, stacksize),
+                  flags=rffi.cast(lltype.Signed, flags),
+                  code=space.str_w(w_code),
+                  consts=space.fixedview(w_consts),
+                  names=unwrap_list_of_strings(space, w_names),
+                  varnames=unwrap_list_of_strings(space, w_varnames),
+                  filename=space.str_w(w_filename),
+                  name=space.str_w(w_funcname),
+                  firstlineno=rffi.cast(lltype.Signed, firstlineno),
+                  lnotab=space.str_w(w_lnotab),
+                  freevars=unwrap_list_of_strings(space, w_freevars),
+                  cellvars=unwrap_list_of_strings(space, w_cellvars))
 
 @cpython_api([CONST_STRING, CONST_STRING, rffi.INT_real], PyCodeObject)
 def PyCode_NewEmpty(space, filename, funcname, firstlineno):
     """Creates a new empty code object with the specified source location."""
-    return space.wrap(PyCode(space,
-                             argcount=0,
-                             nlocals=0,
-                             stacksize=0,
-                             flags=0,
-                             code="",
-                             consts=[],
-                             names=[],
-                             varnames=[],
-                             filename=rffi.charp2str(filename),
-                             name=rffi.charp2str(funcname),
-                             firstlineno=rffi.cast(lltype.Signed, firstlineno),
-                             lnotab="",
-                             freevars=[],
-                             cellvars=[]))
+    return PyCode(space,
+                  argcount=0,
+                  nlocals=0,
+                  stacksize=0,
+                  flags=0,
+                  code="",
+                  consts=[],
+                  names=[],
+                  varnames=[],
+                  filename=rffi.charp2str(filename),
+                  name=rffi.charp2str(funcname),
+                  firstlineno=rffi.cast(lltype.Signed, firstlineno),
+                  lnotab="",
+                  freevars=[],
+                  cellvars=[])
 
 @cpython_api([PyCodeObject], Py_ssize_t, error=CANNOT_FAIL)
 def PyCode_GetNumFree(space, w_co):
diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py
--- a/pypy/module/cpyext/import_.py
+++ b/pypy/module/cpyext/import_.py
@@ -20,35 +20,35 @@
     # Get the builtins from current globals
     if caller is not None:
         w_globals = caller.get_w_globals()
-        w_builtin = space.getitem(w_globals, space.wrap('__builtins__'))
+        w_builtin = space.getitem(w_globals, space.newtext('__builtins__'))
     else:
         # No globals -- use standard builtins, and fake globals
         w_builtin = space.getbuiltinmodule('__builtin__')
         w_globals = space.newdict()
-        space.setitem(w_globals, space.wrap("__builtins__"), w_builtin)
+        space.setitem(w_globals, space.newtext("__builtins__"), w_builtin)
 
     # Get the __import__ function from the builtins
     if space.isinstance_w(w_builtin, space.w_dict):
-        w_import = space.getitem(w_builtin, space.wrap("__import__"))
+        w_import = space.getitem(w_builtin, space.newtext("__import__"))
     else:
-        w_import = space.getattr(w_builtin, space.wrap("__import__"))
+        w_import = space.getattr(w_builtin, space.newtext("__import__"))
 
     # Call the __import__ function with the proper argument list
     # Always use absolute import here.
     return space.call_function(w_import,
                                w_name, w_globals, w_globals,
-                               space.newlist([space.wrap("__doc__")]))
+                               space.newlist([space.newtext("__doc__")]))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyImport_ImportModule(space, name):
-    return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
+    return PyImport_Import(space, space.newtext(rffi.charp2str(name)))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyImport_ImportModuleNoBlock(space, name):
     space.warn(
-        space.wrap('PyImport_ImportModuleNoBlock() is not non-blocking'),
+        space.newtext('PyImport_ImportModuleNoBlock() is not non-blocking'),
         space.w_RuntimeWarning)
-    return PyImport_Import(space, space.wrap(rffi.charp2str(name)))
+    return PyImport_Import(space, space.newtext(rffi.charp2str(name)))
 
 @cpython_api([PyObject], PyObject)
 def PyImport_ReloadModule(space, w_mod):
@@ -72,8 +72,8 @@
     modulename = rffi.charp2str(name)
     w_mod = check_sys_modules_w(space, modulename)
     if not w_mod or space.is_w(w_mod, space.w_None):
-        w_mod = Module(space, space.wrap(modulename))
-        space.setitem(space.sys.get('modules'), space.wrap(modulename), w_mod)
+        w_mod = Module(space, space.newtext(modulename))
+        space.setitem(space.sys.get('modules'), space.newtext(modulename), w_mod)
     # return a borrowed ref --- assumes one copy in sys.modules
     return w_mod
 
@@ -116,11 +116,11 @@
     """Like PyImport_ExecCodeModule(), but the __file__ attribute of
     the module object is set to pathname if it is non-NULL."""
     code = space.interp_w(PyCode, w_code)
-    w_name = space.wrap(rffi.charp2str(name))
+    w_name = space.newtext(rffi.charp2str(name))
     if pathname:
         pathname = rffi.charp2str(pathname)
     else:
         pathname = code.co_filename
     w_mod = importing.add_module(space, w_name)
-    space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname))
+    space.setattr(w_mod, space.newtext('__file__'), space.newtext(pathname))
     return importing.exec_code_module(space, w_mod, code, w_name)
diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -53,7 +53,7 @@
     """Create a new integer object with a value of ival.
 
     """
-    return space.wrap(ival)
+    return space.newint(ival)
 
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyInt_AsLong(space, w_obj):
@@ -127,8 +127,8 @@
     LONG_MAX, a long integer object is returned.
     """
     if ival <= LONG_MAX:
-        return space.wrap(intmask(ival))
-    return space.wrap(ival)
+        return space.newint(intmask(ival))
+    return space.newint(ival)
 
 @cpython_api([Py_ssize_t], PyObject)
 def PyInt_FromSsize_t(space, ival):
@@ -136,7 +136,7 @@
     than LONG_MAX or smaller than LONG_MIN, a long integer object is
     returned.
     """
-    return space.wrap(ival)
+    return space.newint(ival)
 
 @cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject)
 def PyInt_FromString(space, str, pend, base):
@@ -154,8 +154,8 @@
     returned.  If overflow warnings are not being suppressed, NULL will be
     returned in this case."""
     s = rffi.charp2str(str)
-    w_str = space.wrap(s)
-    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    w_str = space.newtext(s)
+    w_base = space.newint(rffi.cast(lltype.Signed, base))
     if pend:
         pend[0] = rffi.ptradd(str, len(s))
     return space.call_function(space.w_int, w_str, w_base)
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -40,7 +40,7 @@
 def PyIter_Check(space, w_obj):
     """Return true if the object o supports the iterator protocol."""
     try:
-        w_attr = space.getattr(space.type(w_obj), space.wrap("next"))
+        w_attr = space.getattr(space.type(w_obj), space.newtext("next"))
     except:
         return False
     else:
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -84,7 +84,7 @@
     """Insert the item item into list list in front of index index.  Return
     0 if successful; return -1 and set an exception if unsuccessful.
     Analogous to list.insert(index, item)."""
-    space.call_method(space.w_list, "insert", w_list, space.wrap(index), w_item)
+    space.call_method(space.w_list, "insert", w_list, space.newint(index), w_item)
     return 0
 
 @cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
@@ -134,8 +134,8 @@
     and high.  Return NULL and set an exception if unsuccessful.  Analogous
     to list[low:high].  Negative indices, as when slicing from Python, are not
     supported."""
-    w_start = space.wrap(low)
-    w_stop = space.wrap(high)
+    w_start = space.newint(low)
+    w_stop = space.newint(high)
     return space.getslice(w_list, w_start, w_stop)
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
@@ -145,8 +145,8 @@
     be NULL, indicating the assignment of an empty list (slice deletion).
     Return 0 on success, -1 on failure.  Negative indices, as when
     slicing from Python, are not supported."""
-    w_start = space.wrap(low)
-    w_stop = space.wrap(high)
+    w_start = space.newint(low)
+    w_stop = space.newint(high)
     if w_sequence:
         space.setslice(w_list, w_start, w_stop, w_sequence)
     else:
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -128,7 +128,7 @@
     except OperationError as e:
         if not e.match(space, space.w_OverflowError):
             raise
-    if space.is_true(space.gt(w_long, space.wrap(0))):
+    if space.is_true(space.gt(w_long, space.newint(0))):
         overflow_ptr[0] = rffi.cast(rffi.INT_real, 1)
     else:
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
@@ -149,7 +149,7 @@
     except OperationError as e:
         if not e.match(space, space.w_OverflowError):
             raise
-    if space.is_true(space.gt(w_long, space.wrap(0))):
+    if space.is_true(space.gt(w_long, space.newint(0))):
         overflow_ptr[0] = rffi.cast(rffi.INT_real, 1)
     else:
         overflow_ptr[0] = rffi.cast(rffi.INT_real, -1)
@@ -158,7 +158,7 @@
 @cpython_api([lltype.Float], PyObject)
 def PyLong_FromDouble(space, val):
     """Return a new PyLongObject object from v, or NULL on failure."""
-    return space.long(space.wrap(val))
+    return space.long(space.newfloat(val))
 
 @cpython_api([PyObject], lltype.Float, error=-1.0)
 def PyLong_AsDouble(space, w_long):
@@ -179,8 +179,8 @@
     between 2 and 36, inclusive.  Leading spaces are ignored.  If there are
     no digits, ValueError will be raised."""
     s = rffi.charp2str(str)
-    w_str = space.wrap(s)
-    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    w_str = space.newtext(s)
+    w_base = space.newint(rffi.cast(lltype.Signed, base))
     if pend:
         pend[0] = rffi.ptradd(str, len(s))
     return space.call_function(space.w_long, w_str, w_base)
@@ -192,8 +192,8 @@
     string, length gives the number of characters, and base is the radix
     for the conversion.  The radix must be in the range [2, 36]; if it is
     out of range, ValueError will be raised."""
-    w_value = space.wrap(rffi.wcharpsize2unicode(u, length))
-    w_base = space.wrap(rffi.cast(lltype.Signed, base))
+    w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
+    w_base = space.newint(rffi.cast(lltype.Signed, base))
     return space.call_function(space.w_long, w_value, w_base)
 
 @cpython_api([rffi.VOIDP], PyObject)
@@ -205,7 +205,7 @@
     value = rffi.cast(ADDR, p)    # signed integer
     if value < 0:
         return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p))
-    return space.wrap(value)
+    return space.newint(value)
 
 @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO))
 def PyLong_AsVoidPtr(space, w_long):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -92,14 +92,14 @@
                 raise oefmt(space.w_TypeError,
                             "%s() takes exactly one argument (%d given)",
                             self.name, length)
-            w_arg = space.getitem(w_args, space.wrap(0))
+            w_arg = space.getitem(w_args, space.newint(0))
             return generic_cpy_call(space, func, w_self, w_arg)
         elif flags & METH_VARARGS:
             return generic_cpy_call(space, func, w_self, w_args)
         else: # METH_OLDARGS, the really old style
             size = length
             if size == 1:
-                w_arg = space.getitem(w_args, space.wrap(0))
+                w_arg = space.getitem(w_args, space.newint(0))
             elif size == 0:
                 w_arg = None
             else:
@@ -109,7 +109,7 @@
     def get_doc(self, space):
         doc = self.ml.c_ml_doc
         if doc:
-            return space.wrap(rffi.charp2str(rffi.cast(rffi.CCHARP,doc)))
+            return space.newtext(rffi.charp2str(rffi.cast(rffi.CCHARP,doc)))
         else:
             return space.w_None
 
@@ -193,9 +193,9 @@
         return self.wrapper_func(space, w_self, w_args, func_to_call)
 
     def descr_method_repr(self):
-        return self.space.wrap("<slot wrapper '%s' of '%s' objects>" %
-                               (self.method_name,
-                                self.w_objclass.name))
+        return self.space.newtext("<slot wrapper '%s' of '%s' objects>" %
+                                  (self.method_name,
+                                   self.w_objclass.name))
 
 def cwrapper_descr_call(space, w_self, __args__):
     self = space.interp_w(W_PyCWrapperObject, w_self)
@@ -204,7 +204,7 @@
     w_self = args_w[0]
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     return self.call(space, w_self, w_args, w_kw)
 
 
@@ -214,7 +214,7 @@
     w_args = space.newtuple(args_w)
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     ret = self.call(space, None, w_args, w_kw)
     return ret
 
@@ -225,7 +225,7 @@
     w_args = space.newtuple(args_w[1:])
     w_kw = space.newdict()
     for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.wrap(key), w_obj)
+        space.setitem(w_kw, space.newtext(key), w_obj)
     ret = self.call(space, w_instance, w_args, w_kw)
     return ret
 
@@ -234,14 +234,14 @@
                         not space.is_w(w_obj, space.w_None) or
                         space.is_w(w_cls, space.type(space.w_None)))
     if asking_for_bound:
-        return space.wrap(Method(space, w_function, w_obj, w_cls))
+        return Method(space, w_function, w_obj, w_cls)
     else:
         return w_function
 
 def cclassmethod_descr_get(space, w_function, w_obj, w_cls=None):
     if not w_cls:
         w_cls = space.type(w_obj)
-    return space.wrap(Method(space, w_function, w_cls, space.w_None))
+    return Method(space, w_function, w_cls, space.w_None)
 
 
 W_PyCFunctionObject.typedef = TypeDef(
@@ -290,7 +290,7 @@
 
 @cpython_api([lltype.Ptr(PyMethodDef), PyObject, PyObject], PyObject)
 def PyCFunction_NewEx(space, ml, w_self, w_name):
-    return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name))
+    return W_PyCFunctionObject(space, ml, w_self, w_name)
 
 @cpython_api([PyObject], PyCFunction_typedef)
 def PyCFunction_GetFunction(space, w_obj):
@@ -305,19 +305,19 @@
 
 @cpython_api([PyObject], PyObject)
 def PyStaticMethod_New(space, w_func):
-    return space.wrap(StaticMethod(w_func))
+    return StaticMethod(w_func)
 
 @cpython_api([PyObject], PyObject)
 def PyClassMethod_New(space, w_func):
-    return space.wrap(ClassMethod(w_func))
+    return ClassMethod(w_func)
 
 @cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
 def PyDescr_NewMethod(space, w_type, method):
-    return space.wrap(W_PyCMethodObject(space, method, w_type))
+    return W_PyCMethodObject(space, method, w_type)
 
 @cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
 def PyDescr_NewClassMethod(space, w_type, method):
-    return space.wrap(W_PyCClassMethodObject(space, method, w_type))
+    return W_PyCClassMethodObject(space, method, w_type)
 
 @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject)
 def Py_FindMethod(space, table, w_obj, name_ptr):
@@ -341,10 +341,10 @@
                 break
             if name == "__methods__":
                 method_list_w.append(
-                    space.wrap(rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name))))
+                    space.newtext(rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name))))
             elif rffi.charp2str(rffi.cast(rffi.CCHARP, method.c_ml_name)) == name: # XXX expensive copy
-                return space.wrap(W_PyCFunctionObject(space, method, w_obj))
+                return W_PyCFunctionObject(space, method, w_obj)
     if name == "__methods__":
         return space.newlist(method_list_w)
-    raise OperationError(space.w_AttributeError, space.wrap(name))
+    raise OperationError(space.w_AttributeError, space.newtext(name))
 
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -21,12 +21,12 @@
     loaded, you will get an empty module object. Use PyImport_ImportModule()
     or one of its variants to import a module.  Package structures implied by a
     dotted name for name are not created if not already present."""
-    w_name = space.wrap(name)
+    w_name = space.newtext(name)
     w_modules = space.sys.get('modules')
 
     w_mod = space.finditem_str(w_modules, name)
     if w_mod is None:
-        w_mod = space.wrap(Module(space, w_name))
+        w_mod = Module(space, w_name)
         space.setitem(w_modules, w_name, w_mod)
 
     return w_mod
@@ -65,15 +65,15 @@
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_self, modname)
     for key, w_value in dict_w.items():
-        space.setattr(w_mod, space.wrap(key), w_value)
+        space.setattr(w_mod, space.newtext(key), w_value)
     if doc:
-        space.setattr(w_mod, space.wrap("__doc__"),
-                      space.wrap(rffi.charp2str(doc)))
+        space.setattr(w_mod, space.newtext("__doc__"),
+                      space.newtext(rffi.charp2str(doc)))
     return w_mod   # borrowed result kept alive in PyImport_AddModule()
 
 
 def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None):
-    w_name = space.wrap(name)
+    w_name = space.newtext_or_none(name)
     methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods)
     if methods:
         i = -1
@@ -90,7 +90,7 @@
                     raise oefmt(space.w_ValueError,
                             "module functions cannot set METH_CLASS or "
                             "METH_STATIC")
-                w_obj = space.wrap(W_PyCFunctionObject(space, method, w_self, w_name))
+                w_obj = W_PyCFunctionObject(space, method, w_self, w_name)
             else:
                 if methodname in dict_w and not (flags & METH_COEXIST):
                     continue
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -119,7 +119,7 @@
     value on success, or NULL on failure. This is the equivalent of the Python
     expression o.attr_name."""
     name = rffi.charp2str(name_ptr)
-    return space.getattr(w_obj, space.wrap(name))
+    return space.getattr(w_obj, space.newtext(name))
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyObject_HasAttr(space, w_obj, w_name):
@@ -133,7 +133,7 @@
 def PyObject_HasAttrString(space, w_obj, name_ptr):
     try:
         name = rffi.charp2str(name_ptr)
-        w_res = operation.hasattr(space, w_obj, space.wrap(name))
+        w_res = operation.hasattr(space, w_obj, space.newtext(name))
         return space.is_true(w_res)
     except OperationError:
         return 0
@@ -145,7 +145,7 @@
 
 @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1)
 def PyObject_SetAttrString(space, w_obj, name_ptr, w_value):
-    w_name = space.wrap(rffi.charp2str(name_ptr))
+    w_name = space.newtext(rffi.charp2str(name_ptr))
     operation.setattr(space, w_obj, w_name, w_value)
     return 0
 
@@ -160,7 +160,7 @@
 def PyObject_DelAttrString(space, w_obj, name_ptr):
     """Delete attribute named attr_name, for object o. Returns -1 on failure.
     This is the equivalent of the Python statement del o.attr_name."""
-    w_name = space.wrap(rffi.charp2str(name_ptr))
+    w_name = space.newtext(rffi.charp2str(name_ptr))
     space.delattr(w_obj, w_name)
     return 0
 
@@ -235,7 +235,7 @@
 @cpython_api([PyObject], PyObject)
 def PyObject_Str(space, w_obj):
     if w_obj is None:
-        return space.wrap("<NULL>")
+        return space.newtext("<NULL>")
     return space.str(w_obj)
 
 @cpython_api([PyObject], PyObject)
@@ -245,13 +245,13 @@
     Python expression repr(o).  Called by the repr() built-in function and
     by reverse quotes."""
     if w_obj is None:
-        return space.wrap("<NULL>")
+        return space.newtext("<NULL>")
     return space.repr(w_obj)
 
 @cpython_api([PyObject, PyObject], PyObject)
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
-        w_format_spec = space.wrap('')
+        w_format_spec = space.newtext('')
     return space.call_method(w_obj, '__format__', w_format_spec)
 
 @cpython_api([PyObject], PyObject)
@@ -261,7 +261,7 @@
     the Python expression unicode(o).  Called by the unicode() built-in
     function."""
     if w_obj is None:
-        return space.wrap(u"<NULL>")
+        return space.newunicode(u"<NULL>")
     return space.call_function(space.w_unicode, w_obj)
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
@@ -385,7 +385,7 @@
         fd = space.int_w(w_obj)
     except OperationError:
         try:
-            w_meth = space.getattr(w_obj, space.wrap('fileno'))
+            w_meth = space.getattr(w_obj, space.newtext('fileno'))
         except OperationError:
             raise oefmt(space.w_TypeError,
                         "argument must be an int, or have a fileno() method.")
@@ -409,7 +409,7 @@
 
 @cpython_api([rffi.DOUBLE], rffi.LONG, error=-1)
 def _Py_HashDouble(space, v):
-    return space.int_w(space.hash(space.wrap(v)))
+    return space.int_w(space.hash(space.newfloat(v)))
 
 @cpython_api([PyObject], lltype.Signed, error=-1)
 def PyObject_HashNotImplemented(space, o):
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -21,7 +21,7 @@
 @cpython_api([PyObject, CONST_STRING], lltype.Void)
 def PyErr_SetString(space, w_type, message_ptr):
     message = rffi.charp2str(message_ptr)
-    PyErr_SetObject(space, w_type, space.wrap(message))
+    PyErr_SetObject(space, w_type, space.newtext(message))
 
 @cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL)
 def PyErr_SetNone(space, w_type):
@@ -58,7 +58,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+        ptraceback[0] = make_ref(space, operror.get_traceback())
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
@@ -170,13 +170,13 @@
     msg = os.strerror(errno)
     if w_value:
         w_error = space.call_function(w_type,
-                                      space.wrap(errno),
-                                      space.wrap(msg),
+                                      space.newint(errno),
+                                      space.newtext(msg),
                                       w_value)
     else:
         w_error = space.call_function(w_type,
-                                      space.wrap(errno),
-                                      space.wrap(msg))
+                                      space.newint(errno),
+                                      space.newtext(msg))
     raise OperationError(w_type, w_error)
 
 @cpython_api([], rffi.INT_real, error=-1)
@@ -252,11 +252,11 @@
     documentation.  There is no C API for warning control."""
     if w_category is None:
         w_category = space.w_None
-    w_message = space.wrap(rffi.charp2str(message_ptr))
-    w_stacklevel = space.wrap(rffi.cast(lltype.Signed, stacklevel))
+    w_message = space.newtext(rffi.charp2str(message_ptr))
+    w_stacklevel = space.newint(rffi.cast(lltype.Signed, stacklevel))
 
-    w_module = PyImport_Import(space, space.wrap("warnings"))
-    w_warn = space.getattr(w_module, space.wrap("warn"))
+    w_module = PyImport_Import(space, space.newtext("warnings"))
+    w_warn = space.getattr(w_module, space.newtext("warn"))
     space.call_function(w_warn, w_message, w_category, w_stacklevel)
     return 0
 
@@ -286,7 +286,7 @@
 
     w_type = operror.w_type
     w_value = operror.get_w_value(space)
-    w_tb = space.wrap(operror.get_traceback())
+    w_tb = operror.get_traceback()
 
     if rffi.cast(lltype.Signed, set_sys_last_vars):
         space.sys.setdictvalue(space, "last_type", w_type)
@@ -317,10 +317,10 @@
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
 def PyTraceBack_Print(space, w_tb, w_file):
-    space.call_method(w_file, "write", space.wrap(
+    space.call_method(w_file, "write", space.newtext(
         'Traceback (most recent call last):\n'))
     w_traceback = space.call_method(space.builtin, '__import__',
-                                    space.wrap("traceback"))
+                                    space.newtext("traceback"))
     space.call_method(w_traceback, "print_tb", w_tb, space.w_None, w_file)
     return 0
 
@@ -373,7 +373,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+        ptraceback[0] = make_ref(space, operror.get_traceback())
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -22,15 +22,15 @@
     0, however, one line is read regardless of length, but EOFError is
     raised if the end of the file is reached immediately."""
     try:
-        w_readline = space.getattr(w_obj, space.wrap('readline'))
+        w_readline = space.getattr(w_obj, space.newtext('readline'))
     except OperationError:
         raise oefmt(space.w_TypeError,
             "argument must be a file, or have a readline() method.")
 
     n = rffi.cast(lltype.Signed, n)
-    if space.is_true(space.gt(space.wrap(n), space.wrap(0))):
-        return space.call_function(w_readline, space.wrap(n))
-    elif space.is_true(space.lt(space.wrap(n), space.wrap(0))):
+    if space.is_true(space.gt(space.newint(n), space.newint(0))):
+        return space.call_function(w_readline, space.newint(n))
+    elif space.is_true(space.lt(space.newint(n), space.newint(0))):
         return space.call_function(w_readline)
     else:
         # XXX Raise EOFError as specified
@@ -117,7 +117,7 @@
 @cpython_api([PyObject], PyObject)
 def PyFile_Name(space, w_p):
     """Return the name of the file specified by p as a string object."""
-    w_name = space.getattr(w_p, space.wrap("name"))
+    w_name = space.getattr(w_p, space.newtext("name"))
     return w_name     # borrowed ref, should be a W_StringObject from the file
 
 @cpython_api([PyObject, rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL)
@@ -137,8 +137,8 @@
             w_newflag = space.w_True
         else:
             w_newflag = space.w_False
-        oldflag = space.int_w(space.getattr(w_p, space.wrap("softspace")))
-        space.setattr(w_p, space.wrap("softspace"), w_newflag)
+        oldflag = space.int_w(space.getattr(w_p, space.newtext("softspace")))
+        space.setattr(w_p, space.newtext("softspace"), w_newflag)
         return oldflag
     except OperationError as e:
         return 0
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -19,7 +19,7 @@
     """Return the result of repeating sequence object o count times, or NULL on
     failure.  This is the equivalent of the Python expression o * count.
     """
-    return space.mul(w_obj, space.wrap(count))
+    return space.mul(w_obj, space.newint(count))
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PySequence_Check(space, w_obj):
@@ -55,7 +55,7 @@
         return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+            raise OperationError(space.w_TypeError, space.newtext(rffi.charp2str(m)))
         raise e
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
@@ -107,20 +107,20 @@
 def PySequence_GetSlice(space, w_obj, start, end):
     """Return the slice of sequence object o between i1 and i2, or NULL on
     failure. This is the equivalent of the Python expression o[i1:i2]."""
-    return space.getslice(w_obj, space.wrap(start), space.wrap(end))
+    return space.getslice(w_obj, space.newint(start), space.newint(end))
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PySequence_SetSlice(space, w_obj, start, end, w_value):
     """Assign the sequence object v to the slice in sequence object o from i1 to
     i2.  This is the equivalent of the Python statement o[i1:i2] = v."""
-    space.setslice(w_obj, space.wrap(start), space.wrap(end), w_value)
+    space.setslice(w_obj, space.newint(start), space.newint(end), w_value)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], rffi.INT_real, error=-1)
 def PySequence_DelSlice(space, w_obj, start, end):
     """Delete the slice in sequence object o from i1 to i2.  Returns -1 on
     failure.  This is the equivalent of the Python statement del o[i1:i2]."""
-    space.delslice(w_obj, space.wrap(start), space.wrap(end))
+    space.delslice(w_obj, space.newint(start), space.newint(end))
     return 0
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject)
@@ -132,7 +132,7 @@
 
     This function used an int type for i. This might require
     changes in your code for properly supporting 64-bit systems."""
-    return space.getitem(w_obj, space.wrap(i))
+    return space.getitem(w_obj, space.newint(i))
 
 @cpython_api([PyObject, Py_ssize_t], PyObject)
 def PySequence_GetItem(space, w_obj, i):
@@ -175,7 +175,7 @@
 
     This function used an int type for count. This might require
     changes in your code for properly supporting 64-bit systems."""
-    return space.inplace_mul(w_o, space.wrap(count))
+    return space.inplace_mul(w_o, space.newint(count))
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
@@ -203,14 +203,14 @@
     if PyDict_Check(space, w_o) or not PySequence_Check(space, w_o):
         raise oefmt(space.w_TypeError,
                     "'%T' object does not support item assignment", w_o)
-    space.setitem(w_o, space.wrap(i), w_v)
+    space.setitem(w_o, space.newint(i), w_v)
     return 0
 
 @cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1)
 def PySequence_DelItem(space, w_o, i):
     """Delete the ith element of object o.  Returns -1 on failure.  This is the
     equivalent of the Python statement del o[i]."""
-    space.delitem(w_o, space.wrap(i))
+    space.delitem(w_o, space.newint(i))
     return 0
 
 @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -133,7 +133,7 @@
     res = rffi.cast(lltype.Signed, res)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(bool(res))
+    return space.newbool(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
@@ -222,7 +222,7 @@
 def wrap_lenfunc(space, w_self, w_args, func):
     func_len = rffi.cast(lenfunc, func)
     check_num_args(space, w_args, 0)
-    return space.wrap(generic_cpy_call(space, func_len, w_self))
+    return space.newint(generic_cpy_call(space, func_len, w_self))
 
 def wrap_sq_item(space, w_self, w_args, func):
     func_target = rffi.cast(ssizeargfunc, func)
@@ -259,7 +259,7 @@
     res = rffi.cast(lltype.Signed, res)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(bool(res))
+    return space.newbool(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -302,7 +302,7 @@
     res = generic_cpy_call(space, func_target, w_self)
     if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.newint(res)
 
 class CPyBuffer(Buffer):
     # Similar to Py_buffer
@@ -422,7 +422,7 @@
                     "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'",
                     w_self, w_self, w_other)
 
-    return space.wrap(generic_cpy_call(space, func_target, w_self, w_other))
+    return space.newint(generic_cpy_call(space, func_target, w_self, w_other))
 
 from rpython.rlib.nonconst import NonConstant
 
@@ -513,7 +513,7 @@
             @cpython_api([PyObject, Py_ssize_t], PyObject, header=header)
             @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
             def slot_func(space, w_self, arg):
-                return space.call_function(slot_fn, w_self, space.wrap(arg))
+                return space.call_function(slot_fn, w_self, space.newint(arg))
             api_func = slot_func.api_func
             handled = True
 
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -99,7 +99,7 @@
             space = self.space
             argv = space.sys.get('argv')
             if space.len_w(argv):
-                argv0 = space.getitem(argv, space.wrap(0))
+                argv0 = space.getitem(argv, space.newint(0))
                 progname = space.str_w(argv0)
             else:
                 progname = "pypy"
@@ -137,7 +137,7 @@
         if not isinstance(w_mod, Module):
             msg = "fixup_extension: module '%s' not loaded" % name
             raise OperationError(space.w_SystemError,
-                                 space.wrap(msg))
+                                 space.newtext(msg))
         w_dict = w_mod.getdict(space)
         w_copy = space.call_method(w_dict, 'copy')
         self.extensions[path] = w_copy
diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -45,13 +45,15 @@
         if typ == member_type:
             result = rffi.cast(rffi.CArrayPtr(lltyp), addr)
             if lltyp is rffi.FLOAT:
-                w_result = space.wrap(lltype.cast_primitive(lltype.Float,
+                w_result = space.newfloat(lltype.cast_primitive(lltype.Float,
                                                             result[0]))
             elif typ == T_BOOL:
                 x = rffi.cast(lltype.Signed, result[0])
-                w_result = space.wrap(x != 0)
+                w_result = space.newbool(x != 0)
+            elif typ == T_DOUBLE:
+                w_result = space.newfloat(result[0])
             else:
-                w_result = space.wrap(result[0])
+                w_result = space.newint(result[0])
             return w_result
 
     if member_type == T_STRING:
@@ -77,7 +79,7 @@
         if obj_ptr[0]:
             w_result = from_ref(space, obj_ptr[0])
         else:
-            w_name = space.wrap(rffi.charp2str(w_member.c_name))
+            w_name = space.newtext(rffi.charp2str(w_member.c_name))
             raise OperationError(space.w_AttributeError, w_name)
     else:
         raise oefmt(space.w_SystemError, "bad memberdescr type")
@@ -98,7 +100,7 @@
     elif w_value is None:
         if member_type == T_OBJECT_EX:
             if not rffi.cast(PyObjectP, addr)[0]:
-                w_name = space.wrap(rffi.charp2str(w_member.c_name))
+                w_name = space.newtext(rffi.charp2str(w_member.c_name))
                 raise OperationError(space.w_AttributeError, w_name)
         elif member_type != T_OBJECT:
             raise oefmt(space.w_TypeError,
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -210,4 +210,4 @@
     """Take a slice of the tuple pointed to by p from low to high and return it
     as a new tuple.
     """
-    return space.getslice(w_obj, space.wrap(low), space.wrap(high))
+    return space.getslice(w_obj, space.newint(low), space.newint(high))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -403,9 +403,9 @@
     set."""
     if not encoding:
         raise oefmt(space.w_TypeError, "decoding Unicode is not supported")
-    w_encoding = space.wrap(rffi.charp2str(encoding))
+    w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
+        w_errors = space.newtext(rffi.charp2str(errors))
     else:
         w_errors = None
 
@@ -415,7 +415,7 @@
         w_meth = None
     else:
         try:
-            w_meth = space.getattr(w_obj, space.wrap('decode'))
+            w_meth = space.getattr(w_obj, space.newtext('decode'))
         except OperationError as e:
             if not e.match(space, space.w_AttributeError):
                 raise
@@ -428,7 +428,7 @@
 def PyUnicode_FromString(space, s):
     """Create a Unicode object from an UTF-8 encoded null-terminated char buffer"""
     w_str = space.newbytes(rffi.charp2str(s))
-    return space.call_method(w_str, 'decode', space.wrap("utf-8"))
+    return space.call_method(w_str, 'decode', space.newtext("utf-8"))
 
 @cpython_api([CONST_STRING, Py_ssize_t], PyObject, result_is_ll=True)
 def PyUnicode_FromStringAndSize(space, s, size):
@@ -451,7 +451,7 @@
     The ordinal must be in range(0x10000) on narrow Python builds
     (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
     raised in case it is not."""
-    w_ordinal = space.wrap(rffi.cast(lltype.Signed, ordinal))
+    w_ordinal = space.newint(rffi.cast(lltype.Signed, ordinal))
     return space.call_function(space.builtin.get('unichr'), w_ordinal)
 
 @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1)
@@ -497,10 +497,10 @@
         """
         w_s = space.newbytes(rffi.charpsize2str(s, size))
         if errors:
-            w_errors = space.wrap(rffi.charp2str(errors))
+            w_errors = space.newtext(rffi.charp2str(errors))
         else:
             w_errors = None
-        return space.call_method(w_s, 'decode', space.wrap(encoding), w_errors)
+        return space.call_method(w_s, 'decode', space.newtext(encoding), w_errors)
     globals()['PyUnicode_Decode%s' % suffix] = PyUnicode_DecodeXXX
 
     @cpython_api([CONST_WSTRING, Py_ssize_t, CONST_STRING], PyObject)
@@ -509,12 +509,12 @@
         """Encode the Py_UNICODE buffer of the given size and return a
         Python string object.  Return NULL if an exception was raised
         by the codec."""
-        w_u = space.wrap(rffi.wcharpsize2unicode(s, size))
+        w_u = space.newunicode(rffi.wcharpsize2unicode(s, size))
         if errors:
-            w_errors = space.wrap(rffi.charp2str(errors))
+            w_errors = space.newtext(rffi.charp2str(errors))
         else:
             w_errors = None
-        return space.call_method(w_u, 'encode', space.wrap(encoding), w_errors)
+        return space.call_method(w_u, 'encode', space.newtext(encoding), w_errors)
     globals()['PyUnicode_Encode%s' % suffix] = PyUnicode_EncodeXXX
 
 make_conversion_functions('UTF8', 'utf-8')
@@ -575,7 +575,7 @@
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
 
-    return space.wrap(result)
+    return space.newunicode(result)
 
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
@@ -631,7 +631,7 @@
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
 
-    return space.wrap(result)
+    return space.newunicode(result)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP],
              rffi.INT_real, error=-1)
@@ -701,7 +701,7 @@
     return the resulting Unicode object. maxcount == -1 means replace all
     occurrences."""
     return space.call_method(w_str, "replace", w_substr, w_replstr,
-                             space.wrap(maxcount))
+                             space.newint(maxcount))
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
              rffi.INT_real, error=-1)
@@ -721,7 +721,7 @@
     """Return the number of non-overlapping occurrences of substr in
     str[start:end].  Return -1 if an error occurred."""
     w_count = space.call_method(w_str, "count", w_substr,
-                                space.wrap(start), space.wrap(end))
+                                space.newint(start), space.newint(end))
     return space.int_w(w_count)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
@@ -735,10 +735,10 @@
     has been set."""
     if rffi.cast(lltype.Signed, direction) > 0:
         w_pos = space.call_method(w_str, "find", w_substr,
-                                  space.wrap(start), space.wrap(end))
+                                  space.newint(start), space.newint(end))
     else:
         w_pos = space.call_method(w_str, "rfind", w_substr,
-                                  space.wrap(start), space.wrap(end))
+                                  space.newint(start), space.newint(end))
     return space.int_w(w_pos)
 
 @cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
@@ -750,7 +750,7 @@
     are not included in the resulting list."""
     if w_sep is None:
         w_sep = space.w_None
-    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+    return space.call_method(w_str, "split", w_sep, space.newint(maxsplit))
 
 @cpython_api([PyObject, rffi.INT_real], PyObject)
 def PyUnicode_Splitlines(space, w_str, keepend):
@@ -758,4 +758,4 @@
     Unicode strings.  CRLF is considered to be one line break.  If
     keepend is 0, the Line break characters are not included in the
     resulting strings."""
-    return space.call_method(w_str, "splitlines", space.wrap(keepend))
+    return space.call_method(w_str, "splitlines", space.newbool(bool(keepend)))

From pypy.commits at gmail.com  Thu Nov  3 05:51:12 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 02:51:12 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _sre
Message-ID: <581b0890.45ad1c0a.8d011.093f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88110:65ea20add619
Date: 2016-11-03 10:50 +0100
http://bitbucket.org/pypy/pypy/changeset/65ea20add619/

Log:	_sre

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -19,11 +19,11 @@
 
 @unwrap_spec(char_ord=int, flags=int)
 def w_getlower(space, char_ord, flags):
-    return space.wrap(getlower(char_ord, flags))
+    return space.newint(getlower(char_ord, flags))
 
 
 def w_getcodesize(space):
-    return space.wrap(CODESIZE)
+    return space.newint(CODESIZE)
 
 # use the same version of unicodedb as the standard objspace
 import pypy.objspace.std.unicodeobject
@@ -74,20 +74,20 @@
 
 def import_re(space):
     w_builtin = space.getbuiltinmodule('__builtin__')
-    w_import = space.getattr(w_builtin, space.wrap("__import__"))
-    return space.call_function(w_import, space.wrap("re"))
+    w_import = space.getattr(w_builtin, space.newtext("__import__"))
+    return space.call_function(w_import, space.newtext("re"))
 
 def matchcontext(space, ctx):
     try:
         return rsre_core.match_context(ctx)
     except rsre_core.Error as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(e.msg))
+        raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
 def searchcontext(space, ctx):
     try:
         return rsre_core.search_context(ctx)
     except rsre_core.Error as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(e.msg))
+        raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
 # ____________________________________________________________
 #
@@ -160,7 +160,7 @@
             if not searchcontext(space, ctx):
                 break
             num_groups = self.num_groups
-            w_emptystr = space.wrap("")
+            w_emptystr = space.newtext("")
             if num_groups == 0:
                 w_item = slice_w(space, ctx, ctx.match_start, ctx.match_end,
                                  w_emptystr)
@@ -183,7 +183,7 @@
         # scanner() method.
         ctx = self.make_ctx(w_string, pos, endpos)
         scanner = W_SRE_Scanner(self, ctx)
-        return self.space.wrap(scanner)
+        return scanner
 
     @unwrap_spec(maxsplit=int)
     def split_w(self, w_string, maxsplit=0):
@@ -223,7 +223,7 @@
     def subn_w(self, w_repl, w_string, count=0):
         w_item, n = self.subx(w_repl, w_string, count)
         space = self.space
-        return space.newtuple([w_item, space.wrap(n)])
+        return space.newtuple([w_item, space.newint(n)])
 
     def subx(self, w_ptemplate, w_string, count):
         space = self.space
@@ -259,7 +259,7 @@
                 # not a literal; hand it over to the template compiler
                 w_re = import_re(space)
                 w_filter = space.call_method(w_re, '_subx',
-                                             space.wrap(self), w_ptemplate)
+                                             self, w_ptemplate)
                 filter_is_callable = space.is_true(space.callable(w_filter))
         #
         # XXX this is a bit of a mess, but it improves performance a lot
@@ -377,7 +377,7 @@
 def SRE_Pattern__new__(space, w_subtype, w_pattern, flags, w_code,
               groups=0, w_groupindex=None, w_indexgroup=None):
     n = space.len_w(w_code)
-    code = [intmask(space.uint_w(space.getitem(w_code, space.wrap(i))))
+    code = [intmask(space.uint_w(space.getitem(w_code, space.newint(i))))
             for i in range(n)]
     #
     w_srepat = space.allocate_instance(W_SRE_Pattern, w_subtype)
@@ -474,22 +474,22 @@
     def expand_w(self, w_template):
         space = self.space
         w_re = import_re(space)
-        return space.call_method(w_re, '_expand', space.wrap(self.srepat),
-                                 space.wrap(self), w_template)
+        return space.call_method(w_re, '_expand', self.srepat,
+                                 self, w_template)
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def start_w(self, w_groupnum):
-        return self.space.wrap(self.do_span(w_groupnum)[0])
+        return self.space.newint(self.do_span(w_groupnum)[0])
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def end_w(self, w_groupnum):
-        return self.space.wrap(self.do_span(w_groupnum)[1])
+        return self.space.newint(self.do_span(w_groupnum)[1])
 
     @unwrap_spec(w_groupnum=WrappedDefault(0))
     def span_w(self, w_groupnum):
         start, end = self.do_span(w_groupnum)
-        return self.space.newtuple([self.space.wrap(start),
-                                    self.space.wrap(end)])
+        return self.space.newtuple([self.space.newint(start),
+                                    self.space.newint(end)])
 
     def flatten_marks(self):
         if self.flatten_cache is None:
@@ -533,7 +533,7 @@
         if lastindex < 0:
             return space.w_None
         w_result = space.finditem(self.srepat.w_indexgroup,
-                                  space.wrap(lastindex))
+                                  space.newint(lastindex))
         if w_result is None:
             return space.w_None
         return w_result
@@ -541,14 +541,14 @@
     def fget_lastindex(self, space):
         lastindex = self._last_index()
         if lastindex >= 0:
-            return space.wrap(lastindex)
+            return space.newint(lastindex)
         return space.w_None
 
     def fget_pos(self, space):
-        return space.wrap(self.ctx.original_pos)
+        return space.newint(self.ctx.original_pos)
 
     def fget_endpos(self, space):
-        return space.wrap(self.ctx.end)
+        return space.newint(self.ctx.end)
 
     def fget_regs(self, space):
         space = self.space
@@ -556,11 +556,11 @@
         num_groups = self.srepat.num_groups
         result_w = [None] * (num_groups + 1)
         ctx = self.ctx
-        result_w[0] = space.newtuple([space.wrap(ctx.match_start),
-                                      space.wrap(ctx.match_end)])
+        result_w[0] = space.newtuple([space.newint(ctx.match_start),
+                                      space.newint(ctx.match_end)])
         for i in range(num_groups):
-            result_w[i + 1] = space.newtuple([space.wrap(fmarks[i*2]),
-                                              space.wrap(fmarks[i*2+1])])
+            result_w[i + 1] = space.newtuple([space.newint(fmarks[i*2]),
+                                              space.newint(fmarks[i*2+1])])
         return space.newtuple(result_w)
 
     def fget_string(self, space):
@@ -612,7 +612,7 @@
         # or matching succeeded so far.
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.ctx.match_start > self.ctx.end:
@@ -638,7 +638,7 @@
             nextstart += (ctx.match_start == nextstart)
             self.ctx = ctx.fresh_copy(nextstart)
             match = W_SRE_Match(self.srepat, ctx)
-            return self.space.wrap(match)
+            return match
         else:
             self.ctx.match_start += 1     # obscure corner case
             return None

From pypy.commits at gmail.com  Thu Nov  3 06:45:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 03:45:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: silence one test on big endian (the
 same is done on little endian)
Message-ID: <581b1536.c10ec20a.734a5.7e53@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88111:cc0f4adda6f9
Date: 2016-11-03 11:42 +0100
http://bitbucket.org/pypy/pypy/changeset/cc0f4adda6f9/

Log:	silence one test on big endian (the same is done on little endian)

diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -2,6 +2,7 @@
 from ctypes.test import need_symbol, xfail
 import unittest
 import os
+import sys
 
 import ctypes
 import _ctypes_test
@@ -279,7 +280,6 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
 
-    @xfail
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
         # Issue #23319
@@ -295,5 +295,11 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
 
+    # see issue #1213, on big endian it fails for the little endian case
+    if sys.byteorder == 'little':
+        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
+    elif sys.byteorder == 'big':
+        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+
 if __name__ == "__main__":
     unittest.main()

From pypy.commits at gmail.com  Thu Nov  3 07:22:56 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:22:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: use wrap_fsdecoded helper
Message-ID: <581b1e10.ca57c20a.b99fd.4c8d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88112:2b2b3d0f8866
Date: 2016-11-03 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/2b2b3d0f8866/

Log:	use wrap_fsdecoded helper

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -760,7 +760,7 @@
         cur = os.getlogin()
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.fsdecode(space.newbytes(cur))
+    return space.wrap_fsdecoded(cur)
 
 # ____________________________________________________________
 
@@ -882,8 +882,7 @@
         if _WIN32:
             result_w[i] = space.wrap(result[i])
         else:
-            w_bytes = space.newbytes(result[i])
-            result_w[i] = space.fsdecode(w_bytes)
+            result_w[i] = space.wrap_fsdecoded(result[i])
     return space.newlist(result_w)
 
 @unwrap_spec(fd=c_int)
@@ -1543,7 +1542,7 @@
         r = os.uname()
     except OSError as e:
         raise wrap_oserror(space, e)
-    l_w = [space.fsdecode(space.newbytes(i))
+    l_w = [space.wrap_fsdecoded(i)
            for i in [r[0], r[1], r[2], r[3], r[4]]]
     w_tuple = space.newtuple(l_w)
     w_uname_result = space.getattr(space.getbuiltinmodule(os.name),
@@ -1858,7 +1857,7 @@
 @unwrap_spec(fd=c_int)
 def ttyname(space, fd):
     try:
-        return space.fsdecode(space.newbytes(os.ttyname(fd)))
+        return space.wrap_fsdecoded(os.ttyname(fd))
     except OSError as e:
         raise wrap_oserror(space, e)
 
@@ -2047,7 +2046,7 @@
 
     Return the name of the controlling terminal for this process.
     """
-    return space.fsdecode(space.newbytes(os.ctermid()))
+    return space.wrap_fsdecoded(os.ctermid())
 
 @unwrap_spec(fd=c_int)
 def device_encoding(space, fd):

From pypy.commits at gmail.com  Thu Nov  3 07:26:07 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:26:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: add missing docstrings
Message-ID: <581b1ecf.94841c0a.aa1c0.15a3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88113:eaecb5690cd9
Date: 2016-11-03 12:25 +0100
http://bitbucket.org/pypy/pypy/changeset/eaecb5690cd9/

Log:	add missing docstrings

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -179,11 +179,19 @@
 @jit.dont_look_inside
 @unwrap_spec(timeout=int)
 def alarm(space, timeout):
+    """alarm(seconds)
+
+    Arrange for SIGALRM to arrive after the given number of seconds.
+    """
     return space.wrap(c_alarm(timeout))
 
 
 @jit.dont_look_inside
 def pause(space):
+    """pause()
+
+    Wait until a signal arrives.
+    """
     c_pause()
     return space.w_None
 
@@ -258,6 +266,12 @@
 @jit.dont_look_inside
 @unwrap_spec(signum=int, flag=int)
 def siginterrupt(space, signum, flag):
+    """siginterrupt(sig, flag) -> None
+
+    change system call restart behaviour: if flag is False, system calls
+    will be restarted when interrupted by signal sig, else system calls
+    will be interrupted.
+    """
     check_signum_in_range(space, signum)
     if rffi.cast(lltype.Signed, c_siginterrupt(signum, flag)) < 0:
         errno = rposix.get_saved_errno()

From pypy.commits at gmail.com  Thu Nov  3 07:45:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 04:45:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: skip tests on non x86 cpus (vmprof is
 not implemented there)
Message-ID: <581b236b.8ca71c0a.ff099.25d5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88114:3beef9de1045
Date: 2016-11-03 11:54 +0100
http://bitbucket.org/pypy/pypy/changeset/3beef9de1045/

Log:	skip tests on non x86 cpus (vmprof is not implemented there)

diff --git a/rpython/rlib/rvmprof/test/__init__.py b/rpython/rlib/rvmprof/test/__init__.py
--- a/rpython/rlib/rvmprof/test/__init__.py
+++ b/rpython/rlib/rvmprof/test/__init__.py
@@ -0,0 +1,5 @@
+import pytest
+import platform
+
+if not platform.machine().startswith('x86'):
+    pytest.skip()

From pypy.commits at gmail.com  Thu Nov  3 07:45:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 04:45:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: add minimal support for vmprof on s390x
 (not in the jit), passes 2 tests
Message-ID: <581b236d.4b8f1c0a.9ef4b.b177@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88115:929b9079f0e9
Date: 2016-11-03 12:44 +0100
http://bitbucket.org/pypy/pypy/changeset/929b9079f0e9/

Log:	add minimal support for vmprof on s390x (not in the jit), passes 2
	tests

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,4 +1,3 @@
-
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 
diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -1,4 +1,3 @@
-
 class AppTestFaultHandler:
     spaceconfig = {
         "usemodules": ["faulthandler", "_vmprof"]
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -30,9 +30,6 @@
 
 
 def setup():
-    if host_platform.machine() == 's390x':
-        raise VMProfPlatformUnsupported("rvmprof not supported on"
-                                        " s390x CPUs for now")
     compile_extra = ['-DRPYTHON_LL2CTYPES']
     platform.verify_eci(ExternalCompilationInfo(
         compile_extra=compile_extra,
diff --git a/rpython/rlib/rvmprof/src/vmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
--- a/rpython/rlib/rvmprof/src/vmprof_config.h
+++ b/rpython/rlib/rvmprof/src/vmprof_config.h
@@ -22,6 +22,8 @@
   #define PC_FROM_UCONTEXT uc_mcontext.arm_ip
 #elif defined(__linux) && defined(__i386) && defined(__GNUC__)
   #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_EIP]
+#elif defined(__s390x__)
+  #define PC_FROM_UCONTEXT uc_mcontext.psw.addr
 #else
   /* linux, gnuc */
   #define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]

From pypy.commits at gmail.com  Thu Nov  3 07:56:03 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:56:03 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: termios
Message-ID: <581b25d3.a285c20a.fcc1f.9ddf@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88116:d07a5af67b43
Date: 2016-11-03 12:12 +0100
http://bitbucket.org/pypy/pypy/changeset/d07a5af67b43/

Log:	termios

diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -14,14 +14,14 @@
         suffixes_w.append(
             space.newtuple([space.newtext(importing.get_so_extension(space)),
                             space.newtext('rb'),
-                            space.newtext(importing.C_EXTENSION)]))
+                            space.newint(importing.C_EXTENSION)]))
     suffixes_w.extend([
         space.newtuple([space.newtext('.py'),
                         space.newtext('U'),
-                        space.newtext(importing.PY_SOURCE)]),
+                        space.newint(importing.PY_SOURCE)]),
         space.newtuple([space.newtext('.pyc'),
                         space.newtext('rb'),
-                        space.newtext(importing.PY_COMPILED)]),
+                        space.newint(importing.PY_COMPILED)]),
         ])
     return space.newlist(suffixes_w)
 
diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py
--- a/pypy/module/termios/interp_termios.py
+++ b/pypy/module/termios/interp_termios.py
@@ -29,7 +29,7 @@
     for w_c in space.unpackiterable(w_cc):
         if space.isinstance_w(w_c, space.w_int):
             ch = space.call_function(space.getattr(w_builtin,
-                                          space.wrap('chr')), w_c)
+                                          space.newtext('chr')), w_c)
             cc.append(space.str_w(ch))
         else:
             cc.append(space.str_w(w_c))
@@ -48,12 +48,12 @@
     except OSError as e:
         raise convert_error(space, e)
     iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tup
-    l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]]
+    l_w = [space.newint(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]]
     # last one need to be chosen carefully
-    cc_w = [space.wrap(i) for i in cc]
+    cc_w = [space.newbytes(i) for i in cc]
     if lflag & rtermios.ICANON:
-        cc_w[rtermios.VMIN] = space.wrap(ord(cc[rtermios.VMIN][0]))
-        cc_w[rtermios.VTIME] = space.wrap(ord(cc[rtermios.VTIME][0]))
+        cc_w[rtermios.VMIN] = space.newint(ord(cc[rtermios.VMIN][0]))
+        cc_w[rtermios.VTIME] = space.newint(ord(cc[rtermios.VTIME][0]))
     w_cc = space.newlist(cc_w)
     l_w.append(w_cc)
     return space.newlist(l_w)

From pypy.commits at gmail.com  Thu Nov  3 07:56:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:56:05 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _locale
Message-ID: <581b25d5.c3181c0a.35782.26b3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88117:e2a03c55685b
Date: 2016-11-03 12:13 +0100
http://bitbucket.org/pypy/pypy/changeset/e2a03c55685b/

Log:	_locale

diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -13,36 +13,36 @@
 import sys
 
 def make_error(space, msg):
-    return OperationError(space.gettypeobject(W_Error.typedef), space.wrap(msg))
+    return OperationError(space.gettypeobject(W_Error.typedef), space.newtext(msg))
 
 def rewrap_error(space, e):
     return OperationError(space.gettypeobject(W_Error.typedef),
-                          space.wrap(e.message))
+                          space.newtext(e.message))
 
 def _fixup_ulcase(space):
     stringmod = space.call_function(
         space.getattr(space.getbuiltinmodule('__builtin__'),
-                      space.wrap('__import__')), space.wrap('string'))
+                      space.newtext('__import__')), space.newtext('string'))
     # create uppercase map string
     ul = []
     for c in xrange(256):
         if rlocale.isupper(c):
             ul.append(chr(c))
-    space.setattr(stringmod, space.wrap('uppercase'), space.wrap(''.join(ul)))
+    space.setattr(stringmod, space.newtext('uppercase'), space.newbytes(''.join(ul)))
 
     # create lowercase string
     ul = []
     for c in xrange(256):
         if rlocale.islower(c):
             ul.append(chr(c))
-    space.setattr(stringmod, space.wrap('lowercase'), space.wrap(''.join(ul)))
+    space.setattr(stringmod, space.newtext('lowercase'), space.newbytes(''.join(ul)))
 
     # create letters string
     ul = []
     for c in xrange(256):
         if rlocale.isalpha(c):
             ul.append(chr(c))
-    space.setattr(stringmod, space.wrap('letters'), space.wrap(''.join(ul)))
+    space.setattr(stringmod, space.newtext('letters'), space.newbytes(''.join(ul)))
 
 @unwrap_spec(category=int)
 def setlocale(space, category, w_locale=None):
@@ -61,12 +61,12 @@
     if category in (rlocale.LC_CTYPE, rlocale.LC_ALL):
         _fixup_ulcase(space)
 
-    return space.wrap(result)
+    return space.newtext(result)
 
 def _w_copy_grouping(space, text):
-    groups = [ space.wrap(ord(group)) for group in text ]
+    groups = [ space.newint(ord(group)) for group in text ]
     if groups:
-        groups.append(space.wrap(0))
+        groups.append(space.newint(0))
     return space.newlist(groups)
 
 def localeconv(space):
@@ -75,43 +75,42 @@
 
     # Numeric information
     w_result = space.newdict()
-    w = space.wrap
-    space.setitem(w_result, w("decimal_point"),
-                  w(rffi.charp2str(lp.c_decimal_point)))
-    space.setitem(w_result, w("thousands_sep"),
-                  w(rffi.charp2str(lp.c_thousands_sep)))
-    space.setitem(w_result, w("grouping"),
+    space.setitem(w_result, space.newtext("decimal_point"),
+                  space.newtext(rffi.charp2str(lp.c_decimal_point)))
+    space.setitem(w_result, space.newtext("thousands_sep"),
+                  space.newtext(rffi.charp2str(lp.c_thousands_sep)))
+    space.setitem(w_result, space.newtext("grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_grouping)))
-    space.setitem(w_result, w("int_curr_symbol"),
-                  w(rffi.charp2str(lp.c_int_curr_symbol)))
-    space.setitem(w_result, w("currency_symbol"),
-                  w(rffi.charp2str(lp.c_currency_symbol)))
-    space.setitem(w_result, w("mon_decimal_point"),
-                  w(rffi.charp2str(lp.c_mon_decimal_point)))
-    space.setitem(w_result, w("mon_thousands_sep"),
-                  w(rffi.charp2str(lp.c_mon_thousands_sep)))
-    space.setitem(w_result, w("mon_grouping"),
+    space.setitem(w_result, space.newtext("int_curr_symbol"),
+                  space.newtext(rffi.charp2str(lp.c_int_curr_symbol)))
+    space.setitem(w_result, space.newtext("currency_symbol"),
+                  space.newtext(rffi.charp2str(lp.c_currency_symbol)))
+    space.setitem(w_result, space.newtext("mon_decimal_point"),
+                  space.newtext(rffi.charp2str(lp.c_mon_decimal_point)))
+    space.setitem(w_result, space.newtext("mon_thousands_sep"),
+                  space.newtext(rffi.charp2str(lp.c_mon_thousands_sep)))
+    space.setitem(w_result, space.newtext("mon_grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_mon_grouping)))
-    space.setitem(w_result, w("positive_sign"),
-                  w(rffi.charp2str(lp.c_positive_sign)))
-    space.setitem(w_result, w("negative_sign"),
-                  w(rffi.charp2str(lp.c_negative_sign)))
-    space.setitem(w_result, w("int_frac_digits"),
-                  w(lp.c_int_frac_digits))
-    space.setitem(w_result, w("frac_digits"),
-                  w(lp.c_frac_digits))
-    space.setitem(w_result, w("p_cs_precedes"),
-                  w(lp.c_p_cs_precedes))
-    space.setitem(w_result, w("p_sep_by_space"),
-                  w(lp.c_p_sep_by_space))
-    space.setitem(w_result, w("n_cs_precedes"),
-                  w(lp.c_n_cs_precedes))
-    space.setitem(w_result, w("n_sep_by_space"),
-                  w(lp.c_n_sep_by_space))
-    space.setitem(w_result, w("p_sign_posn"),
-                  w(lp.c_p_sign_posn))
-    space.setitem(w_result, w("n_sign_posn"),
-                  w(lp.c_n_sign_posn))
+    space.setitem(w_result, space.newtext("positive_sign"),
+                  space.newtext(rffi.charp2str(lp.c_positive_sign)))
+    space.setitem(w_result, space.newtext("negative_sign"),
+                  space.newtext(rffi.charp2str(lp.c_negative_sign)))
+    space.setitem(w_result, space.newtext("int_frac_digits"),
+                  space.newint(lp.c_int_frac_digits))
+    space.setitem(w_result, space.newtext("frac_digits"),
+                  space.newint(lp.c_frac_digits))
+    space.setitem(w_result, space.newtext("p_cs_precedes"),
+                  space.newint(lp.c_p_cs_precedes))
+    space.setitem(w_result, space.newtext("p_sep_by_space"),
+                  space.newint(lp.c_p_sep_by_space))
+    space.setitem(w_result, space.newtext("n_cs_precedes"),
+                  space.newint(lp.c_n_cs_precedes))
+    space.setitem(w_result, space.newtext("n_sep_by_space"),
+                  space.newint(lp.c_n_sep_by_space))
+    space.setitem(w_result, space.newtext("p_sign_posn"),
+                  space.newint(lp.c_p_sign_posn))
+    space.setitem(w_result, space.newtext("n_sign_posn"),
+                  space.newint(lp.c_n_sign_posn))
 
     return w_result
 
@@ -129,7 +128,7 @@
         s1_c = rffi.str2charp(s1)
         s2_c = rffi.str2charp(s2)
         try:
-            return space.wrap(_strcoll(s1_c, s2_c))
+            return space.newint(_strcoll(s1_c, s2_c))
         finally:
             rffi.free_charp(s1_c)
             rffi.free_charp(s2_c)
@@ -144,7 +143,7 @@
         rffi.free_wcharp(s1_c)
         rffi.free_wcharp(s2_c)
 
-    return space.wrap(result)
+    return space.newint(result)
 
 _strxfrm = rlocale.external('strxfrm',
                     [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
@@ -174,7 +173,7 @@
     val = rffi.charp2str(buf)
     lltype.free(buf, flavor="raw")
 
-    return space.wrap(val)
+    return space.newtext(val)
 
 if rlocale.HAVE_LANGINFO:
 
@@ -184,7 +183,7 @@
         Return the value for the locale information associated with key."""
 
         try:
-            return space.wrap(rlocale.nl_langinfo(key))
+            return space.newtext(rlocale.nl_langinfo(key))
         except ValueError:
             raise oefmt(space.w_ValueError, "unsupported langinfo constant")
 
@@ -200,7 +199,7 @@
         Return translation of msg."""
         msg_c = rffi.str2charp(msg)
         try:
-            return space.wrap(rffi.charp2str(_gettext(msg_c)))
+            return space.newtext(rffi.charp2str(_gettext(msg_c)))
         finally:
             rffi.free_charp(msg_c)
 
@@ -235,7 +234,7 @@
                 rffi.free_charp(domain_c)
                 rffi.free_charp(msg_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
     _dcgettext = rlocale.external('dcgettext', [rffi.CCHARP, rffi.CCHARP, rffi.INT],
                                                                 rffi.CCHARP)
@@ -271,7 +270,7 @@
                 rffi.free_charp(domain_c)
                 rffi.free_charp(msg_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
 
     _textdomain = rlocale.external('textdomain', [rffi.CCHARP], rffi.CCHARP)
@@ -296,7 +295,7 @@
             finally:
                 rffi.free_charp(domain_c)
 
-        return space.wrap(result)
+        return space.newtext(result)
 
     _bindtextdomain = rlocale.external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP],
                                                                 rffi.CCHARP,
@@ -326,8 +325,8 @@
 
         if not dirname:
             errno = rposix.get_saved_errno()
-            raise OperationError(space.w_OSError, space.wrap(errno))
-        return space.wrap(rffi.charp2str(dirname))
+            raise OperationError(space.w_OSError, space.newint(errno))
+        return space.newtext(rffi.charp2str(dirname))
 
     _bind_textdomain_codeset = rlocale.external('bind_textdomain_codeset',
                                     [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP)
@@ -358,9 +357,9 @@
             if not result:
                 return space.w_None
             else:
-                return space.wrap(rffi.charp2str(result))
+                return space.newtext(rffi.charp2str(result))
 
 if sys.platform == 'win32':
     def getdefaultlocale(space):
         language, encoding = rlocale.getdefaultlocale()
-        return space.newtuple([space.wrap(language), space.wrap(encoding)])
+        return space.newtuple([space.newtext(language), space.newtext(encoding)])

From pypy.commits at gmail.com  Thu Nov  3 07:56:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:56:06 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: random
Message-ID: <581b25d6.d4301c0a.ba28a.1800@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88118:7f9a88e99894
Date: 2016-11-03 12:13 +0100
http://bitbucket.org/pypy/pypy/changeset/7f9a88e99894/

Log:	random

diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py
--- a/pypy/module/_random/interp_random.py
+++ b/pypy/module/_random/interp_random.py
@@ -13,7 +13,7 @@
     x = space.allocate_instance(W_Random, w_subtype)
     x = space.interp_w(W_Random, x)
     W_Random.__init__(x, space, w_anything)
-    return space.wrap(x)
+    return x
 
 
 class W_Random(W_Root):
@@ -34,7 +34,7 @@
                 w_n = space.abs(w_n)
             else:
                 n = space.hash_w(w_n)
-                w_n = space.wrap(r_uint(n))
+                w_n = space.newint(r_uint(n))
         key = []
         w_one = space.newint(1)
         w_two = space.newint(2)
@@ -54,7 +54,7 @@
     def getstate(self, space):
         state = [None] * (rrandom.N + 1)
         for i in range(rrandom.N):
-            state[i] = space.wrap(widen(self._rnd.state[i]))
+            state[i] = space.newint(widen(self._rnd.state[i]))
         state[rrandom.N] = space.newlong(self._rnd.index)
         return space.newtuple(state)
 

From pypy.commits at gmail.com  Thu Nov  3 07:56:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:56:08 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: cStringIO
Message-ID: <581b25d8.8a0d1c0a.5bd8.b7e6@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88119:16a7bce25aec
Date: 2016-11-03 12:55 +0100
http://bitbucket.org/pypy/pypy/changeset/16a7bce25aec/

Log:	cStringIO

diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -26,7 +26,7 @@
 
     def descr_getvalue(self):
         self.check_closed()
-        return self.space.wrap(self.getvalue())
+        return self.space.newtext(self.getvalue())
 
     def descr_isatty(self):
         self.check_closed()
@@ -38,17 +38,17 @@
         line = self.readline()
         if len(line) == 0:
             raise OperationError(space.w_StopIteration, space.w_None)
-        return space.wrap(line)
+        return space.newtext(line)
 
     @unwrap_spec(n=int)
     def descr_read(self, n=-1):
         self.check_closed()
-        return self.space.wrap(self.read(n))
+        return self.space.newtext(self.read(n))
 
     @unwrap_spec(size=int)
     def descr_readline(self, size=-1):
         self.check_closed()
-        return self.space.wrap(self.readline(size))
+        return self.space.newtext(self.readline(size))
 
     @unwrap_spec(size=int)
     def descr_readlines(self, size=0):
@@ -58,7 +58,7 @@
             line = self.readline()
             if len(line) == 0:
                 break
-            lines_w.append(self.space.wrap(line))
+            lines_w.append(self.space.newtext(line))
             if size > 0:
                 size -= len(line)
                 if size <= 0:
@@ -76,7 +76,7 @@
 
     def descr_tell(self):
         self.check_closed()
-        return self.space.wrap(self.tell())
+        return self.space.newint(self.tell())
 
     # abstract methods
     def close(self):                  assert False, "abstract"
@@ -183,10 +183,10 @@
 # ____________________________________________________________
 
 def descr_closed(self, space):
-    return space.wrap(self.is_closed())
+    return space.newbool(self.is_closed())
 
 def descr_softspace(self, space):
-    return space.wrap(self.softspace)
+    return space.newint(self.softspace)
 
 def descr_setsoftspace(self, space, w_newvalue):
     self.softspace = space.int_w(w_newvalue)
@@ -236,7 +236,7 @@
 
 def StringIO(space, w_string=None):
     if space.is_none(w_string):
-        return space.wrap(W_OutputType(space))
+        return W_OutputType(space)
     else:
         string = space.getarg_w('s*', w_string).as_str()
-        return space.wrap(W_InputType(space, string))
+        return W_InputType(space, string)

From pypy.commits at gmail.com  Thu Nov  3 07:56:09 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 04:56:09 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: thread
Message-ID: <581b25d9.8ca71c0a.ff099.2a1d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88120:35acc5809da1
Date: 2016-11-03 12:55 +0100
http://bitbucket.org/pypy/pypy/changeset/35acc5809da1/

Log:	thread

diff --git a/pypy/module/thread/error.py b/pypy/module/thread/error.py
--- a/pypy/module/thread/error.py
+++ b/pypy/module/thread/error.py
@@ -6,4 +6,4 @@
 
 def wrap_thread_error(space, msg):
     w_error = space.fromcache(Cache).w_error
-    return OperationError(w_error, space.wrap(msg))
+    return OperationError(w_error, space.newtext(msg))
diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py
--- a/pypy/module/thread/os_local.py
+++ b/pypy/module/thread/os_local.py
@@ -51,7 +51,7 @@
         # call __init__
         try:
             w_type = space.type(self)
-            w_init = space.getattr(w_type, space.wrap("__init__"))
+            w_init = space.getattr(w_type, space.newtext("__init__"))
             space.call_obj_args(w_init, self, self.initargs)
         except:
             # failed, forget w_dict and propagate the exception
@@ -76,7 +76,7 @@
     def descr_local__new__(space, w_subtype, __args__):
         local = space.allocate_instance(Local, w_subtype)
         Local.__init__(local, space, __args__)
-        return space.wrap(local)
+        return local
 
     def descr_local__init__(self, space):
         # No arguments allowed
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -160,4 +160,4 @@
 def allocate_lock(space):
     """Create a new lock object.  (allocate() is an obsolete synonym.)
 See LockType.__doc__ for information about locks."""
-    return space.wrap(Lock(space))
+    return Lock(space)
diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py
--- a/pypy/module/thread/test/test_local.py
+++ b/pypy/module/thread/test/test_local.py
@@ -123,6 +123,7 @@
             return {}
         def wrap(self, obj):
             return obj
+        newtext = wrap
         def type(self, obj):
             return type(obj)
         class config:

From pypy.commits at gmail.com  Thu Nov  3 10:13:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 07:13:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: missing import in ppc backend
Message-ID: <581b461e.8a0d1c0a.5bd8.edad@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88121:69cdab401e30
Date: 2016-11-03 15:12 +0100
http://bitbucket.org/pypy/pypy/changeset/69cdab401e30/

Log:	missing import in ppc backend

diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -30,6 +30,7 @@
 from rpython.jit.backend.ppc import callbuilder
 from rpython.rlib.rarithmetic import r_uint
 from rpython.jit.backend.ppc.vector_ext import VectorAssembler
+from rpython.rlib.rjitlog import rjitlog as jl
 
 class IntOpAssembler(object):
         

From pypy.commits at gmail.com  Thu Nov  3 11:15:37 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 03 Nov 2016 08:15:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-ssl: renamed _ssl to openssl (it is easier
 to build the cffi module without modifying _cffi_src/*)
Message-ID: <581b5499.e8edc20a.1018a.e97b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88122:7e3079c33ec6
Date: 2016-11-03 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/7e3079c33ec6/

Log:	renamed _ssl to openssl (it is easier to build the cffi module
	without modifying _cffi_src/*)

diff too long, truncating to 2000 out of 15778 lines

diff --git a/lib_pypy/_ssl/.gitignore b/lib_pypy/_ssl/.gitignore
deleted file mode 100644
--- a/lib_pypy/_ssl/.gitignore
+++ /dev/null
@@ -1,95 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# dotenv
-.env
-
-# virtualenv
-.venv/
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-
-# Rope project settings
-.ropeproject
-
-# Vim
-
-*.swp
-*.swo
diff --git a/lib_pypy/_ssl/LICENSE b/lib_pypy/_ssl/LICENSE
deleted file mode 100644
--- a/lib_pypy/_ssl/LICENSE
+++ /dev/null
@@ -1,26 +0,0 @@
-
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and
-documentation is licensed as follows: 
-
-    The MIT License
-
-    Permission is hereby granted, free of charge, to any person 
-    obtaining a copy of this software and associated documentation 
-    files (the "Software"), to deal in the Software without 
-    restriction, including without limitation the rights to use, 
-    copy, modify, merge, publish, distribute, sublicense, and/or 
-    sell copies of the Software, and to permit persons to whom the 
-    Software is furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be included 
-    in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-    DEALINGS IN THE SOFTWARE.
-
diff --git a/lib_pypy/_ssl/README.md b/lib_pypy/_ssl/README.md
deleted file mode 100644
--- a/lib_pypy/_ssl/README.md
+++ /dev/null
@@ -1,4 +0,0 @@
-# PyPy's SSL module
-
-Most of the CFFI code is copied from cryptography
-
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -1,3 +1,3 @@
-from _ssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION,
+from openssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION,
         _test_decode_cert, _SSLContext)
-from _ssl._stdssl import *
+from openssl._stdssl import *
diff --git a/lib_pypy/_ssl/_cffi_src/__init__.py b/lib_pypy/_ssl/_cffi_src/__init__.py
deleted file mode 100644
diff --git a/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/build_commoncrypto.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-from _cffi_src.utils import build_ffi_for_binding
-
-
-ffi = build_ffi_for_binding(
-    module_name="_commoncrypto",
-    module_prefix="_cffi_src.commoncrypto.",
-    modules=[
-        "cf",
-        "common_digest",
-        "common_hmac",
-        "common_key_derivation",
-        "common_cryptor",
-        "common_symmetric_key_wrap",
-        "seccertificate",
-        "secimport",
-        "secitem",
-        "seckey",
-        "seckeychain",
-        "secpolicy",
-        "sectransform",
-        "sectrust",
-        "secure_transport",
-    ],
-    extra_link_args=[
-        "-framework", "Security", "-framework", "CoreFoundation"
-    ],
-)
diff --git a/lib_pypy/_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_ssl/_cffi_src/build_constant_time.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/build_constant_time.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-import os
-
-from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
-
-
-with open(os.path.join(
-    os.path.dirname(__file__), "hazmat_src/constant_time.h"
-)) as f:
-    types = f.read()
-
-with open(os.path.join(
-    os.path.dirname(__file__), "hazmat_src/constant_time.c"
-)) as f:
-    functions = f.read()
-
-ffi = build_ffi(
-    module_name="_constant_time",
-    cdef_source=types,
-    verify_source=functions,
-    extra_link_args=extra_link_args(compiler_type()),
-)
diff --git a/lib_pypy/_ssl/_cffi_src/build_openssl.py b/lib_pypy/_ssl/_cffi_src/build_openssl.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/build_openssl.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-import os
-import sys
-
-from openssl._cffi_src.utils import (
-    build_ffi_for_binding, compiler_type, extra_link_args
-)
-
-
-def _get_openssl_libraries(platform):
-    # OpenSSL goes by a different library name on different operating systems.
-    if platform == "darwin":
-        return _osx_libraries(
-            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
-        )
-    elif platform == "win32":
-        if compiler_type() == "msvc":
-            libs = ["libeay32", "ssleay32"]
-        else:
-            libs = ["ssl", "crypto"]
-        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
-    else:
-        # In some circumstances, the order in which these libs are
-        # specified on the linker command-line is significant;
-        # libssl must come before libcrypto
-        # (http://marc.info/?l=openssl-users&m=135361825921871)
-        return ["ssl", "crypto"]
-
-
-def _osx_libraries(build_static):
-    # For building statically we don't want to pass the -lssl or -lcrypto flags
-    if build_static == "1":
-        return []
-    else:
-        return ["ssl", "crypto"]
-
-
-ffi = build_ffi_for_binding(
-    module_name="_openssl",
-    module_prefix="_cffi_src.openssl.",
-    modules=[
-        # This goes first so we can define some cryptography-wide symbols.
-        "cryptography",
-
-        "aes",
-        "asn1",
-        "bignum",
-        "bio",
-        "cmac",
-        "cms",
-        "conf",
-        "crypto",
-        "dh",
-        "dsa",
-        "ec",
-        "ecdh",
-        "ecdsa",
-        "engine",
-        "err",
-        "evp",
-        "hmac",
-        "nid",
-        "objects",
-        "ocsp",
-        "opensslv",
-        "pem",
-        "pkcs12",
-        "rand",
-        "rsa",
-        "ssl",
-        "x509",
-        "x509name",
-        "x509v3",
-        "x509_vfy",
-        "pkcs7",
-        "callbacks",
-    ],
-    libraries=_get_openssl_libraries(sys.platform),
-    extra_link_args=extra_link_args(compiler_type()),
-)
diff --git a/lib_pypy/_ssl/_cffi_src/build_padding.py b/lib_pypy/_ssl/_cffi_src/build_padding.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/build_padding.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-import os
-
-from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
-
-
-with open(os.path.join(
-    os.path.dirname(__file__), "hazmat_src/padding.h"
-)) as f:
-    types = f.read()
-
-with open(os.path.join(
-    os.path.dirname(__file__), "hazmat_src/padding.c"
-)) as f:
-    functions = f.read()
-
-ffi = build_ffi(
-    module_name="_padding",
-    cdef_source=types,
-    verify_source=functions,
-    extra_link_args=extra_link_args(compiler_type()),
-)
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/cf.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CoreFoundation/CoreFoundation.h>
-"""
-
-TYPES = """
-typedef bool Boolean;
-typedef signed long OSStatus;
-typedef unsigned char UInt8;
-typedef uint32_t UInt32;
-
-typedef const void * CFAllocatorRef;
-const CFAllocatorRef kCFAllocatorDefault;
-typedef ... *CFDataRef;
-typedef signed long long CFIndex;
-typedef ... *CFStringRef;
-typedef ... *CFArrayRef;
-typedef ... *CFMutableArrayRef;
-typedef ... *CFBooleanRef;
-typedef ... *CFErrorRef;
-typedef ... *CFNumberRef;
-typedef ... *CFTypeRef;
-typedef ... *CFDictionaryRef;
-typedef ... *CFMutableDictionaryRef;
-typedef struct {
-    ...;
-} CFDictionaryKeyCallBacks;
-typedef struct {
-    ...;
-} CFDictionaryValueCallBacks;
-typedef struct {
-    ...;
-} CFRange;
-typedef struct {
-    ...;
-} CFArrayCallBacks;
-
-typedef UInt32 CFStringEncoding;
-enum {
-    kCFStringEncodingASCII = 0x0600
-};
-
-enum {
-   kCFNumberSInt8Type = 1,
-   kCFNumberSInt16Type = 2,
-   kCFNumberSInt32Type = 3,
-   kCFNumberSInt64Type = 4,
-   kCFNumberFloat32Type = 5,
-   kCFNumberFloat64Type = 6,
-   kCFNumberCharType = 7,
-   kCFNumberShortType = 8,
-   kCFNumberIntType = 9,
-   kCFNumberLongType = 10,
-   kCFNumberLongLongType = 11,
-   kCFNumberFloatType = 12,
-   kCFNumberDoubleType = 13,
-   kCFNumberCFIndexType = 14,
-   kCFNumberNSIntegerType = 15,
-   kCFNumberCGFloatType = 16,
-   kCFNumberMaxType = 16
-};
-typedef int CFNumberType;
-
-const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
-const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
-
-const CFArrayCallBacks kCFTypeArrayCallBacks;
-
-const CFBooleanRef kCFBooleanTrue;
-const CFBooleanRef kCFBooleanFalse;
-"""
-
-FUNCTIONS = """
-CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
-CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
-                                      CFStringEncoding);
-CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
-                                   const void **, CFIndex,
-                                   const CFDictionaryKeyCallBacks *,
-                                   const CFDictionaryValueCallBacks *);
-CFMutableDictionaryRef CFDictionaryCreateMutable(
-    CFAllocatorRef,
-    CFIndex,
-    const CFDictionaryKeyCallBacks *,
-    const CFDictionaryValueCallBacks *
-);
-void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
-CFIndex CFArrayGetCount(CFArrayRef);
-const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
-CFIndex CFDataGetLength(CFDataRef);
-void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
-CFRange CFRangeMake(CFIndex, CFIndex);
-void CFShow(CFTypeRef);
-Boolean CFBooleanGetValue(CFBooleanRef);
-CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
-void CFRelease(CFTypeRef);
-CFTypeRef CFRetain(CFTypeRef);
-
-CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
-                                       const CFArrayCallBacks *);
-void CFArrayAppendValue(CFMutableArrayRef, const void *);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_cryptor.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CommonCrypto/CommonCryptor.h>
-"""
-
-TYPES = """
-enum {
-    kCCAlgorithmAES128 = 0,
-    kCCAlgorithmDES,
-    kCCAlgorithm3DES,
-    kCCAlgorithmCAST,
-    kCCAlgorithmRC4,
-    kCCAlgorithmRC2,
-    kCCAlgorithmBlowfish
-};
-typedef uint32_t CCAlgorithm;
-enum {
-    kCCSuccess = 0,
-    kCCParamError = -4300,
-    kCCBufferTooSmall = -4301,
-    kCCMemoryFailure = -4302,
-    kCCAlignmentError = -4303,
-    kCCDecodeError = -4304,
-    kCCUnimplemented = -4305
-};
-typedef int32_t CCCryptorStatus;
-typedef uint32_t CCOptions;
-enum {
-    kCCEncrypt = 0,
-    kCCDecrypt,
-};
-typedef uint32_t CCOperation;
-typedef ... *CCCryptorRef;
-
-enum {
-    kCCModeOptionCTR_LE = 0x0001,
-    kCCModeOptionCTR_BE = 0x0002
-};
-
-typedef uint32_t CCModeOptions;
-
-enum {
-    kCCModeECB = 1,
-    kCCModeCBC = 2,
-    kCCModeCFB = 3,
-    kCCModeCTR = 4,
-    kCCModeF8 = 5,
-    kCCModeLRW = 6,
-    kCCModeOFB = 7,
-    kCCModeXTS = 8,
-    kCCModeRC4 = 9,
-    kCCModeCFB8 = 10,
-    kCCModeGCM = 11
-};
-typedef uint32_t CCMode;
-enum {
-    ccNoPadding = 0,
-    ccPKCS7Padding = 1,
-};
-typedef uint32_t CCPadding;
-"""
-
-FUNCTIONS = """
-CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
-                                        CCPadding, const void *, const void *,
-                                        size_t, const void *, size_t, int,
-                                        CCModeOptions, CCCryptorRef *);
-CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
-                                const void *, size_t, const void *,
-                                CCCryptorRef *);
-CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
-                                size_t, size_t *);
-CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
-CCCryptorStatus CCCryptorRelease(CCCryptorRef);
-
-CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
-CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
-CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
-                                    void *);
-CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
-                                    void *);
-CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
-CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-/* Not defined in the public header */
-enum {
-    kCCModeGCM = 11
-};
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_digest.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CommonCrypto/CommonDigest.h>
-"""
-
-TYPES = """
-typedef uint32_t CC_LONG;
-typedef uint64_t CC_LONG64;
-typedef struct CC_MD5state_st {
-    ...;
-} CC_MD5_CTX;
-typedef struct CC_SHA1state_st {
-    ...;
-} CC_SHA1_CTX;
-typedef struct CC_SHA256state_st {
-    ...;
-} CC_SHA256_CTX;
-typedef struct CC_SHA512state_st {
-    ...;
-} CC_SHA512_CTX;
-"""
-
-FUNCTIONS = """
-int CC_MD5_Init(CC_MD5_CTX *);
-int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
-int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
-
-int CC_SHA1_Init(CC_SHA1_CTX *);
-int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
-int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
-
-int CC_SHA224_Init(CC_SHA256_CTX *);
-int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
-int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
-
-int CC_SHA256_Init(CC_SHA256_CTX *);
-int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
-int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
-
-int CC_SHA384_Init(CC_SHA512_CTX *);
-int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
-int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
-
-int CC_SHA512_Init(CC_SHA512_CTX *);
-int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
-int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_hmac.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CommonCrypto/CommonHMAC.h>
-"""
-
-TYPES = """
-typedef struct {
-    ...;
-} CCHmacContext;
-enum {
-    kCCHmacAlgSHA1,
-    kCCHmacAlgMD5,
-    kCCHmacAlgSHA256,
-    kCCHmacAlgSHA384,
-    kCCHmacAlgSHA512,
-    kCCHmacAlgSHA224
-};
-typedef uint32_t CCHmacAlgorithm;
-"""
-
-FUNCTIONS = """
-void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
-void CCHmacUpdate(CCHmacContext *, const void *, size_t);
-void CCHmacFinal(CCHmacContext *, void *);
-
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_key_derivation.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CommonCrypto/CommonKeyDerivation.h>
-"""
-
-TYPES = """
-enum {
-    kCCPBKDF2 = 2,
-};
-typedef uint32_t CCPBKDFAlgorithm;
-enum {
-    kCCPRFHmacAlgSHA1 = 1,
-    kCCPRFHmacAlgSHA224 = 2,
-    kCCPRFHmacAlgSHA256 = 3,
-    kCCPRFHmacAlgSHA384 = 4,
-    kCCPRFHmacAlgSHA512 = 5,
-};
-typedef uint32_t CCPseudoRandomAlgorithm;
-typedef unsigned int uint;
-"""
-
-FUNCTIONS = """
-int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
-                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
-                         uint, uint8_t *, size_t);
-uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
-                      CCPseudoRandomAlgorithm, size_t, uint32_t);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <CommonCrypto/CommonSymmetricKeywrap.h>
-"""
-
-TYPES = """
-enum {
-    kCCWRAPAES = 1,
-};
-
-typedef uint32_t CCWrappingAlgorithm;
-"""
-
-FUNCTIONS = """
-int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
-                        const uint8_t *, size_t, const uint8_t *, size_t,
-                        uint8_t *, size_t *);
-int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
-                         const size_t, const uint8_t *, size_t,
-                         const uint8_t *, size_t, uint8_t *, size_t *);
-size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
-size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
-
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/seccertificate.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecCertificate.h>
-"""
-
-TYPES = """
-typedef ... *SecCertificateRef;
-"""
-
-FUNCTIONS = """
-SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/secimport.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecImportExport.h>
-"""
-
-TYPES = """
-typedef ... *SecAccessRef;
-
-CFStringRef kSecImportExportPassphrase;
-CFStringRef kSecImportExportKeychain;
-CFStringRef kSecImportExportAccess;
-
-typedef uint32_t SecExternalItemType;
-enum {
-    kSecItemTypeUnknown,
-    kSecItemTypePrivateKey,
-    kSecItemTypePublicKey,
-    kSecItemTypeSessionKey,
-    kSecItemTypeCertificate,
-    kSecItemTypeAggregate
-};
-
-
-typedef uint32_t SecExternalFormat;
-enum {
-    kSecFormatUnknown = 0,
-    kSecFormatOpenSSL,
-    kSecFormatSSH,
-    kSecFormatBSAFE,
-    kSecFormatRawKey,
-    kSecFormatWrappedPKCS8,
-    kSecFormatWrappedOpenSSL,
-    kSecFormatWrappedSSH,
-    kSecFormatWrappedLSH,
-    kSecFormatX509Cert,
-    kSecFormatPEMSequence,
-    kSecFormatPKCS7,
-    kSecFormatPKCS12,
-    kSecFormatNetscapeCertSequence,
-    kSecFormatSSHv2
-};
-
-typedef uint32_t SecItemImportExportFlags;
-enum {
-    kSecKeyImportOnlyOne        = 0x00000001,
-    kSecKeySecurePassphrase     = 0x00000002,
-    kSecKeyNoAccessControl      = 0x00000004
-};
-typedef uint32_t SecKeyImportExportFlags;
-
-typedef struct {
-    /* for import and export */
-    uint32_t version;
-    SecKeyImportExportFlags  flags;
-    CFTypeRef                passphrase;
-    CFStringRef              alertTitle;
-    CFStringRef              alertPrompt;
-
-    /* for import only */
-    SecAccessRef             accessRef;
-    CFArrayRef               keyUsage;
-
-    CFArrayRef               keyAttributes;
-} SecItemImportExportKeyParameters;
-"""
-
-FUNCTIONS = """
-OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
-                       SecExternalItemType *, SecItemImportExportFlags,
-                       const SecItemImportExportKeyParameters *,
-                       SecKeychainRef, CFArrayRef *);
-OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
-OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
-                       const SecItemImportExportKeyParameters *, CFDataRef *);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/secitem.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecItem.h>
-"""
-
-TYPES = """
-const CFTypeRef kSecAttrKeyType;
-const CFTypeRef kSecAttrKeySizeInBits;
-const CFTypeRef kSecAttrIsPermanent;
-const CFTypeRef kSecAttrKeyTypeRSA;
-const CFTypeRef kSecAttrKeyTypeDSA;
-const CFTypeRef kSecUseKeychain;
-"""
-
-FUNCTIONS = """
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckey.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecKey.h>
-"""
-
-TYPES = """
-typedef ... *SecKeyRef;
-"""
-
-FUNCTIONS = """
-OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
-size_t SecKeyGetBlockSize(SecKeyRef);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/seckeychain.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecKeychain.h>
-"""
-
-TYPES = """
-typedef ... *SecKeychainRef;
-"""
-
-FUNCTIONS = """
-OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
-                           SecAccessRef, SecKeychainRef *);
-OSStatus SecKeychainDelete(SecKeychainRef);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/secpolicy.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecPolicy.h>
-"""
-
-TYPES = """
-typedef ... *SecPolicyRef;
-"""
-
-FUNCTIONS = """
-SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectransform.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecDigestTransform.h>
-#include <Security/SecSignVerifyTransform.h>
-#include <Security/SecEncryptTransform.h>
-"""
-
-TYPES = """
-typedef ... *SecTransformRef;
-
-CFStringRef kSecImportExportPassphrase;
-CFStringRef kSecImportExportKeychain;
-CFStringRef kSecImportExportAccess;
-
-CFStringRef kSecEncryptionMode;
-CFStringRef kSecEncryptKey;
-CFStringRef kSecIVKey;
-CFStringRef kSecModeCBCKey;
-CFStringRef kSecModeCFBKey;
-CFStringRef kSecModeECBKey;
-CFStringRef kSecModeNoneKey;
-CFStringRef kSecModeOFBKey;
-CFStringRef kSecOAEPEncodingParametersAttributeName;
-CFStringRef kSecPaddingKey;
-CFStringRef kSecPaddingNoneKey;
-CFStringRef kSecPaddingOAEPKey;
-CFStringRef kSecPaddingPKCS1Key;
-CFStringRef kSecPaddingPKCS5Key;
-CFStringRef kSecPaddingPKCS7Key;
-
-const CFStringRef kSecTransformInputAttributeName;
-const CFStringRef kSecTransformOutputAttributeName;
-const CFStringRef kSecTransformDebugAttributeName;
-const CFStringRef kSecTransformTransformName;
-const CFStringRef kSecTransformAbortAttributeName;
-
-CFStringRef kSecInputIsAttributeName;
-CFStringRef kSecInputIsPlainText;
-CFStringRef kSecInputIsDigest;
-CFStringRef kSecInputIsRaw;
-
-const CFStringRef kSecDigestTypeAttribute;
-const CFStringRef kSecDigestLengthAttribute;
-const CFStringRef kSecDigestMD5;
-const CFStringRef kSecDigestSHA1;
-const CFStringRef kSecDigestSHA2;
-"""
-
-FUNCTIONS = """
-Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
-                                 CFErrorRef *);
-SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
-SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
-SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
-SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
-CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/sectrust.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecTrust.h>
-"""
-
-TYPES = """
-typedef ... *SecTrustRef;
-typedef uint32_t SecTrustResultType;
-
-enum {
-    kSecTrustResultInvalid,
-    kSecTrustResultProceed,
-    kSecTrustResultDeny,
-    kSecTrustResultUnspecified,
-    kSecTrustResultRecoverableTrustFailure,
-    kSecTrustResultFatalTrustFailure,
-    kSecTrustResultOtherError
-};
-"""
-
-FUNCTIONS = """
-OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
-OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
-"""
-
-MACROS = """
-/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
- * has to go here for compatibility.
- */
-OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/commoncrypto/secure_transport.py
+++ /dev/null
@@ -1,308 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <Security/SecureTransport.h>
-"""
-
-TYPES = """
-typedef ... *SSLContextRef;
-typedef const void *SSLConnectionRef;
-
-typedef enum {
-    kSSLSessionOptionBreakOnServerAuth,
-    kSSLSessionOptionBreakOnCertRequested,
-} SSLSessionOption;
-
-typedef enum {
-    kNeverAuthenticate,
-    kAlwaysAuthenticate,
-    kTryAuthenticate
-} SSLAuthenticate;
-
-typedef enum {
-    kSSLIdle,
-    kSSLHandshake,
-    kSSLConnected,
-    kSSLClosed,
-    kSSLAborted
-} SSLSessionState;
-
-typedef enum {
-    kSSLProtocolUnknown = 0,
-    kSSLProtocol3       = 2,
-    kTLSProtocol1       = 4,
-    /* DEPRECATED on iOS */
-    kSSLProtocol2       = 1,
-    kSSLProtocol3Only   = 3,
-    kTLSProtocol1Only   = 5,
-    kSSLProtocolAll     = 6,
-} SSLProtocol;
-
-typedef UInt32 SSLCipherSuite;
-enum {
-   SSL_NULL_WITH_NULL_NULL =               0x0000,
-   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
-   SSL_RSA_WITH_NULL_SHA =                 0x0002,
-   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
-   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
-   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
-   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
-   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
-   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
-   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
-   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
-   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
-   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
-   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
-   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
-   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
-   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
-   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
-   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
-   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
-   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
-   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
-   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
-   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
-   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
-   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
-   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
-   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
-   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
-   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
-
-   /* TLS addenda using AES, per RFC 3268 */
-   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
-   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
-   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
-   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
-   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
-   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
-   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
-   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
-   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
-   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
-   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
-   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
-
-   /* ECDSA addenda, RFC 4492 */
-   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
-   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
-   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
-   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
-   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
-   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
-   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
-   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
-   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
-   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
-   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
-   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
-   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
-   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
-   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
-   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
-   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
-   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
-   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
-   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
-   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
-   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
-   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
-   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
-   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
-
-   /* TLS 1.2 addenda, RFC 5246 */
-   /* Initial state. */
-   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
-
-   /* Server provided RSA certificate for key exchange. */
-   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
-   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
-   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
-   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
-   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
-   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
-   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
-   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
-
-   /* Server-authenticated (and optionally client-authenticated)
-      Diffie-Hellman. */
-   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
-   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
-   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
-   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
-   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
-   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
-   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
-   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
-   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
-   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
-   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
-   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
-
-   /* Completely anonymous Diffie-Hellman */
-   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
-   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
-   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
-   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
-
-   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
-      for TLS. */
-   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
-   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
-   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
-   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
-   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
-   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
-   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
-   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
-   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
-   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
-   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
-   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
-
-   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
-      HMAC SHA-256/384. */
-   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
-   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
-   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
-   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
-   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
-   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
-   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
-   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
-
-   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
-      SHA-256/384 and AES Galois Counter Mode (GCM) */
-   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
-   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
-   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
-   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
-   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
-   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
-   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
-   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
-
-   /* RFC 5746 - Secure Renegotiation */
-   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
-
-   /*
-   * Tags for SSL 2 cipher kinds that are not specified
-   * for SSL 3.
-   */
-   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
-   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
-   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
-   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
-   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
-};
-
-typedef enum {
-    kSSLClientCertNone,
-    kSSLClientCertRequested,
-    kSSLClientCertSent,
-    kSSLClientCertRejected
-} SSLClientCertificateState;
-
-enum {
-    errSSLProtocol              = -9800,
-    errSSLNegotiation           = -9801,
-    errSSLFatalAlert            = -9802,
-    errSSLWouldBlock            = -9803,
-    errSSLSessionNotFound       = -9804,
-    errSSLClosedGraceful        = -9805,
-    errSSLClosedAbort           = -9806,
-    errSSLXCertChainInvalid     = -9807,
-    errSSLBadCert               = -9808,
-    errSSLCrypto                = -9809,
-    errSSLInternal              = -9810,
-    errSSLModuleAttach          = -9811,
-    errSSLUnknownRootCert       = -9812,
-    errSSLNoRootCert            = -9813,
-    errSSLCertExpired           = -9814,
-    errSSLCertNotYetValid       = -9815,
-    errSSLClosedNoNotify        = -9816,
-    errSSLBufferOverflow        = -9817,
-    errSSLBadCipherSuite        = -9818,
-    errSSLPeerUnexpectedMsg     = -9819,
-    errSSLPeerBadRecordMac      = -9820,
-    errSSLPeerDecryptionFail    = -9821,
-    errSSLPeerRecordOverflow    = -9822,
-    errSSLPeerDecompressFail    = -9823,
-    errSSLPeerHandshakeFail     = -9824,
-    errSSLPeerBadCert           = -9825,
-    errSSLPeerUnsupportedCert   = -9826,
-    errSSLPeerCertRevoked       = -9827,
-    errSSLPeerCertExpired       = -9828,
-    errSSLPeerCertUnknown       = -9829,
-    errSSLIllegalParam          = -9830,
-    errSSLPeerUnknownCA         = -9831,
-    errSSLPeerAccessDenied      = -9832,
-    errSSLPeerDecodeError       = -9833,
-    errSSLPeerDecryptError      = -9834,
-    errSSLPeerExportRestriction = -9835,
-    errSSLPeerProtocolVersion   = -9836,
-    errSSLPeerInsufficientSecurity = -9837,
-    errSSLPeerInternalError     = -9838,
-    errSSLPeerUserCancelled     = -9839,
-    errSSLPeerNoRenegotiation   = -9840,
-    errSSLServerAuthCompleted   = -9841,
-    errSSLClientCertRequested   = -9842,
-    errSSLHostNameMismatch      = -9843,
-    errSSLConnectionRefused     = -9844,
-    errSSLDecryptionFail        = -9845,
-    errSSLBadRecordMac          = -9846,
-    errSSLRecordOverflow        = -9847,
-    errSSLBadConfiguration      = -9848,
-    errSSLLast                  = -9849     /* end of range, to be deleted */
-};
-"""
-
-FUNCTIONS = """
-OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
-OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
-OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
-OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
-
-OSStatus SSLHandshake(SSLContextRef);
-OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
-OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
-OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
-OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
-OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
-OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
-OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
-OSStatus SSLClose(SSLContextRef);
-
-OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
-OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
-OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
-OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
-OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
-OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
-OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
-OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
-
-OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
-OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
-OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
-OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
-OSStatus SSLGetClientCertificateState(SSLContextRef,
-                                      SSLClientCertificateState *);
-OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
-
-OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
-OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
-OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
-"""
-
-MACROS = """
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.c
+++ /dev/null
@@ -1,22 +0,0 @@
-// This file is dual licensed under the terms of the Apache License, Version
-// 2.0, and the BSD License. See the LICENSE file in the root of this
-// repository for complete details.
-
-uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
-                                            uint8_t *b, size_t len_b) {
-    size_t i = 0;
-    uint8_t mismatch = 0;
-    if (len_a != len_b) {
-        return 0;
-    }
-    for (i = 0; i < len_a; i++) {
-        mismatch |= a[i] ^ b[i];
-    }
-
-    /* Make sure any bits set are copied to the lowest bit */
-    mismatch |= mismatch >> 4;
-    mismatch |= mismatch >> 2;
-    mismatch |= mismatch >> 1;
-    /* Now check the low bit to see if it's set */
-    return (mismatch & 1) == 0;
-}
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/hazmat_src/constant_time.h
+++ /dev/null
@@ -1,6 +0,0 @@
-// This file is dual licensed under the terms of the Apache License, Version
-// 2.0, and the BSD License. See the LICENSE file in the root of this
-// repository for complete details.
-
-uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
-                                            size_t);
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// This file is dual licensed under the terms of the Apache License, Version
-// 2.0, and the BSD License. See the LICENSE file in the root of this
-// repository for complete details.
-
-/* Returns the value of the input with the most-significant-bit copied to all
-   of the bits. */
-static uint8_t Cryptography_DUPLICATE_MSB_TO_ALL(uint8_t a) {
-    return (1 - (a >> (sizeof(uint8_t) * 8 - 1))) - 1;
-}
-
-/* This returns 0xFF if a < b else 0x00, but does so in a constant time
-   fashion */
-static uint8_t Cryptography_constant_time_lt(uint8_t a, uint8_t b) {
-    a -= b;
-    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
-}
-
-uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
-                                         uint8_t block_len) {
-    uint8_t i;
-    uint8_t pad_size = data[block_len - 1];
-    uint8_t mismatch = 0;
-    for (i = 0; i < block_len; i++) {
-        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
-        uint8_t b = data[block_len - 1 - i];
-        mismatch |= (mask & (pad_size ^ b));
-    }
-
-    /* Check to make sure the pad_size was within the valid range. */
-    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
-    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
-
-    /* Make sure any bits set are copied to the lowest bit */
-    mismatch |= mismatch >> 4;
-    mismatch |= mismatch >> 2;
-    mismatch |= mismatch >> 1;
-    /* Now check the low bit to see if it's set */
-    return (mismatch & 1) == 0;
-}
-
-uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
-                                            uint8_t block_len) {
-    uint8_t i;
-    uint8_t pad_size = data[block_len - 1];
-    uint8_t mismatch = 0;
-    /* Skip the first one with the pad size */
-    for (i = 1; i < block_len; i++) {
-        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
-        uint8_t b = data[block_len - 1 - i];
-        mismatch |= (mask & b);
-    }
-
-    /* Check to make sure the pad_size was within the valid range. */
-    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
-    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
-
-    /* Make sure any bits set are copied to the lowest bit */
-    mismatch |= mismatch >> 4;
-    mismatch |= mismatch >> 2;
-    mismatch |= mismatch >> 1;
-    /* Now check the low bit to see if it's set */
-    return (mismatch & 1) == 0;
-}
diff --git a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/hazmat_src/padding.h
+++ /dev/null
@@ -1,6 +0,0 @@
-// This file is dual licensed under the terms of the Apache License, Version
-// 2.0, and the BSD License. See the LICENSE file in the root of this
-// repository for complete details.
-
-uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
-uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_ssl/_cffi_src/openssl/__init__.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/openssl/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_ssl/_cffi_src/openssl/aes.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/openssl/aes.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <openssl/aes.h>
-"""
-
-TYPES = """
-static const int Cryptography_HAS_AES_WRAP;
-static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
-
-struct aes_key_st {
-    ...;
-};
-typedef struct aes_key_st AES_KEY;
-"""
-
-FUNCTIONS = """
-int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
-int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
-
-int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
-                 const unsigned char *, unsigned int);
-int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
-                   const unsigned char *, unsigned int);
-"""
-
-MACROS = """
-/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
-   this in 1.0.1+. */
-void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
-                        size_t, const AES_KEY *, unsigned char[],
-                        unsigned char[], unsigned int *);
-"""
-
-CUSTOMIZATIONS = """
-static const long Cryptography_HAS_AES_WRAP = 1;
-#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
-static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
-void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
-                           size_t, const AES_KEY *,
-                           unsigned char[], unsigned char[],
-                           unsigned int *) = NULL;
-#else
-static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
-#endif
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_ssl/_cffi_src/openssl/asn1.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/openssl/asn1.py
+++ /dev/null
@@ -1,164 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <openssl/asn1.h>
-"""
-
-TYPES = """
-typedef int... time_t;
-
-typedef int ASN1_BOOLEAN;
-typedef ... ASN1_INTEGER;
-
-struct asn1_string_st {
-    int length;
-    int type;
-    unsigned char *data;
-    long flags;
-};
-
-typedef struct asn1_string_st ASN1_OCTET_STRING;
-typedef struct asn1_string_st ASN1_IA5STRING;
-typedef struct asn1_string_st ASN1_BIT_STRING;
-typedef struct asn1_string_st ASN1_TIME;
-typedef ... ASN1_OBJECT;
-typedef struct asn1_string_st ASN1_STRING;
-typedef struct asn1_string_st ASN1_UTF8STRING;
-typedef ... ASN1_TYPE;
-typedef ... ASN1_GENERALIZEDTIME;
-typedef ... ASN1_ENUMERATED;
-typedef ... ASN1_ITEM;
-typedef ... ASN1_VALUE;
-
-typedef ... ASN1_ITEM_EXP;
-
-typedef ... ASN1_UTCTIME;
-
-static const int V_ASN1_GENERALIZEDTIME;
-
-static const int MBSTRING_FLAG;
-static const int MBSTRING_ASC;
-static const int MBSTRING_BMP;
-static const int MBSTRING_UTF8;
-static const int MBSTRING_UNIV;
-"""
-
-FUNCTIONS = """
-ASN1_OBJECT *ASN1_OBJECT_new(void);
-void ASN1_OBJECT_free(ASN1_OBJECT *);
-
-/*  ASN1 OBJECT IDENTIFIER */
-ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
-
-/*  ASN1 STRING */
-ASN1_STRING *ASN1_STRING_new(void);
-ASN1_STRING *ASN1_STRING_type_new(int);
-void ASN1_STRING_free(ASN1_STRING *);
-unsigned char *ASN1_STRING_data(ASN1_STRING *);
-int ASN1_STRING_set(ASN1_STRING *, const void *, int);
-
-/*  ASN1 OCTET STRING */
-ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
-void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
-int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
-
-/* ASN1 IA5STRING */
-ASN1_IA5STRING *ASN1_IA5STRING_new(void);
-
-/*  ASN1 INTEGER */
-ASN1_INTEGER *ASN1_INTEGER_new(void);
-void ASN1_INTEGER_free(ASN1_INTEGER *);
-int ASN1_INTEGER_set(ASN1_INTEGER *, long);
-
-/*  ASN1 TIME */
-ASN1_TIME *ASN1_TIME_new(void);
-void ASN1_TIME_free(ASN1_TIME *);
-ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
-                                                   ASN1_GENERALIZEDTIME **);
-ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
-
-/*  ASN1 UTCTIME */
-ASN1_UTCTIME *ASN1_UTCTIME_new(void);
-void ASN1_UTCTIME_free(ASN1_UTCTIME *);
-int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
-ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
-
-/*  ASN1 GENERALIZEDTIME */
-int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
-ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
-void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
-int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
-
-/*  ASN1 ENUMERATED */
-ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
-void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
-int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
-int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
-
-ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
-                          const ASN1_ITEM *);
-int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
-"""
-
-MACROS = """
-/* These became const ASN1_* in 1.1.0 */
-int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
-int ASN1_STRING_type(ASN1_STRING *);
-int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
-long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
-int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
-
-ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
-void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
-
-ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
-void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
-int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
-int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
-int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
-/* This is not a macro, but is const on some versions of OpenSSL */
-int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
-ASN1_TIME *M_ASN1_TIME_dup(void *);
-const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
-
-/* These aren't macros these arguments are all const X on openssl > 1.0.x */
-
-int ASN1_TIME_print(BIO *, ASN1_TIME *);
-int ASN1_STRING_length(ASN1_STRING *);
-ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
-int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
-int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
-
-ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
-int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
-
-ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
-int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
-long ASN1_INTEGER_get(ASN1_INTEGER *);
-
-BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
-ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
-
-/* These isn't a macro the arg is const on openssl 1.0.2+ */
-int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
-int ASN1_UTCTIME_check(ASN1_UTCTIME *);
-
-/* Not a macro, const on openssl 1.0 */
-int ASN1_STRING_set_default_mask_asc(char *);
-
-int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
-ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
-"""
-
-CUSTOMIZATIONS = """
-/* This macro is removed in 1.1.0. We re-add it if required to support
-   pyOpenSSL versions older than whatever resolves
-   https://github.com/pyca/pyopenssl/issues/431 */
-#if !defined(M_ASN1_TIME_dup)
-#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
-#endif
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_ssl/_cffi_src/openssl/bignum.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/openssl/bignum.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <openssl/bn.h>
-"""
-
-TYPES = """
-typedef ... BN_CTX;
-typedef ... BIGNUM;
-typedef int... BN_ULONG;
-"""
-
-FUNCTIONS = """
-BIGNUM *BN_new(void);
-void BN_free(BIGNUM *);
-
-BN_CTX *BN_CTX_new(void);
-void BN_CTX_free(BN_CTX *);
-
-void BN_CTX_start(BN_CTX *);
-BIGNUM *BN_CTX_get(BN_CTX *);
-void BN_CTX_end(BN_CTX *);
-
-BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
-BIGNUM *BN_dup(const BIGNUM *);
-
-int BN_set_word(BIGNUM *, BN_ULONG);
-BN_ULONG BN_get_word(const BIGNUM *);
-
-const BIGNUM *BN_value_one(void);
-
-char *BN_bn2hex(const BIGNUM *);
-int BN_hex2bn(BIGNUM **, const char *);
-int BN_dec2bn(BIGNUM **, const char *);
-
-int BN_bn2bin(const BIGNUM *, unsigned char *);
-BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
-
-int BN_num_bits(const BIGNUM *);
-
-int BN_cmp(const BIGNUM *, const BIGNUM *);
-int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
-int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
-int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
-               BN_CTX *);
-int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
-               BN_CTX *);
-int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
-               BN_CTX *);
-int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
-               BN_CTX *);
-int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-
-int BN_set_bit(BIGNUM *, int);
-int BN_clear_bit(BIGNUM *, int);
-
-int BN_is_bit_set(const BIGNUM *, int);
-
-int BN_mask_bits(BIGNUM *, int);
-"""
-
-MACROS = """
-int BN_num_bytes(const BIGNUM *);
-
-int BN_zero(BIGNUM *);
-int BN_one(BIGNUM *);
-int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
-
-int BN_lshift(BIGNUM *, const BIGNUM *, int);
-int BN_lshift1(BIGNUM *, BIGNUM *);
-
-int BN_rshift(BIGNUM *, BIGNUM *, int);
-int BN_rshift1(BIGNUM *, BIGNUM *);
-"""
-
-CUSTOMIZATIONS = """
-"""
diff --git a/lib_pypy/_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_ssl/_cffi_src/openssl/bio.py
deleted file mode 100644
--- a/lib_pypy/_ssl/_cffi_src/openssl/bio.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-
-from __future__ import absolute_import, division, print_function
-
-INCLUDES = """
-#include <openssl/bio.h>
-"""
-
-TYPES = """
-typedef struct bio_st BIO;
-typedef void bio_info_cb(BIO *, int, const char *, int, long, long);
-typedef ... bio_st;
-typedef ... BIO_METHOD;
-typedef ... BUF_MEM;
-
-static const int BIO_TYPE_MEM;
-static const int BIO_TYPE_FILE;
-static const int BIO_TYPE_FD;
-static const int BIO_TYPE_SOCKET;
-static const int BIO_TYPE_CONNECT;
-static const int BIO_TYPE_ACCEPT;
-static const int BIO_TYPE_NULL;
-static const int BIO_CLOSE;
-static const int BIO_NOCLOSE;
-static const int BIO_TYPE_SOURCE_SINK;
-static const int BIO_CTRL_RESET;
-static const int BIO_CTRL_EOF;
-static const int BIO_CTRL_SET;
-static const int BIO_CTRL_SET_CLOSE;
-static const int BIO_CTRL_FLUSH;
-static const int BIO_CTRL_DUP;
-static const int BIO_CTRL_GET_CLOSE;
-static const int BIO_CTRL_INFO;
-static const int BIO_CTRL_GET;
-static const int BIO_CTRL_PENDING;
-static const int BIO_CTRL_WPENDING;
-static const int BIO_C_FILE_SEEK;
-static const int BIO_C_FILE_TELL;
-static const int BIO_TYPE_NONE;
-static const int BIO_TYPE_NBIO_TEST;
-static const int BIO_TYPE_BIO;
-static const int BIO_TYPE_DESCRIPTOR;
-static const int BIO_FLAGS_READ;
-static const int BIO_FLAGS_WRITE;
-static const int BIO_FLAGS_IO_SPECIAL;
-static const int BIO_FLAGS_RWS;
-static const int BIO_FLAGS_SHOULD_RETRY;
-static const int BIO_TYPE_NULL_FILTER;
-static const int BIO_TYPE_SSL;
-static const int BIO_TYPE_MD;
-static const int BIO_TYPE_BUFFER;
-static const int BIO_TYPE_CIPHER;
-static const int BIO_TYPE_BASE64;
-static const int BIO_TYPE_FILTER;
-"""
-
-FUNCTIONS = """
-int BIO_free(BIO *);
-void BIO_vfree(BIO *);
-void BIO_free_all(BIO *);
-BIO *BIO_push(BIO *, BIO *);
-BIO *BIO_pop(BIO *);
-BIO *BIO_next(BIO *);
-BIO *BIO_find_type(BIO *, int);
-BIO *BIO_new_file(const char *, const char *);
-BIO *BIO_new_fp(FILE *, int);
-BIO *BIO_new_fd(int, int);
-BIO *BIO_new_socket(int, int);
-long BIO_ctrl(BIO *, int, long, void *);
-long BIO_callback_ctrl(
-    BIO *,
-    int,
-    void (*)(struct bio_st *, int, const char *, int, long, long)
-);
-long BIO_int_ctrl(BIO *, int, long, int);
-size_t BIO_ctrl_pending(BIO *);
-size_t BIO_ctrl_wpending(BIO *);
-int BIO_read(BIO *, void *, int);
-int BIO_gets(BIO *, char *, int);
-int BIO_write(BIO *, const void *, int);
-int BIO_puts(BIO *, const char *);
-int BIO_method_type(const BIO *);
-"""
-
-MACROS = """
-/* These added const to BIO_METHOD in 1.1.0 */
-BIO *BIO_new(BIO_METHOD *);
-BIO_METHOD *BIO_s_mem(void);
-BIO_METHOD *BIO_s_file(void);
-BIO_METHOD *BIO_s_fd(void);
-BIO_METHOD *BIO_s_socket(void);
-BIO_METHOD *BIO_s_null(void);
-BIO_METHOD *BIO_f_null(void);
-BIO_METHOD *BIO_f_buffer(void);
-/* BIO_new_mem_buf became const void * in 1.0.2g */
-BIO *BIO_new_mem_buf(void *, int);
-long BIO_set_fd(BIO *, long, int);
-long BIO_get_fd(BIO *, char *);
-long BIO_set_mem_eof_return(BIO *, int);
-long BIO_get_mem_data(BIO *, char **);
-long BIO_set_mem_buf(BIO *, BUF_MEM *, int);
-long BIO_get_mem_ptr(BIO *, BUF_MEM **);
-long BIO_set_fp(BIO *, FILE *, int);
-long BIO_get_fp(BIO *, FILE **);
-long BIO_read_filename(BIO *, char *);
-long BIO_write_filename(BIO *, char *);
-long BIO_append_filename(BIO *, char *);
-long BIO_rw_filename(BIO *, char *);
-int BIO_should_read(BIO *);
-int BIO_should_write(BIO *);
-int BIO_should_io_special(BIO *);
-int BIO_retry_type(BIO *);
-int BIO_should_retry(BIO *);
-int BIO_reset(BIO *);
-int BIO_seek(BIO *, int);
-int BIO_tell(BIO *);
-int BIO_flush(BIO *);

From pypy.commits at gmail.com  Thu Nov  3 11:46:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:46:08 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: pwd
Message-ID: <581b5bc0.ca10c20a.1b72a.f205@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88126:7cdb0a7ebd3a
Date: 2016-11-03 16:13 +0100
http://bitbucket.org/pypy/pypy/changeset/7cdb0a7ebd3a/

Log:	pwd

diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py
--- a/pypy/module/pwd/interp_pwd.py
+++ b/pypy/module/pwd/interp_pwd.py
@@ -73,15 +73,15 @@
 
 def make_struct_passwd(space, pw):
     w_passwd_struct = space.getattr(space.getbuiltinmodule('pwd'),
-                                    space.wrap('struct_passwd'))
+                                    space.newtext('struct_passwd'))
     w_tuple = space.newtuple([
-        space.wrap(rffi.charp2str(pw.c_pw_name)),
-        space.wrap(rffi.charp2str(pw.c_pw_passwd)),
-        space.int(space.wrap(pw.c_pw_uid)),
-        space.int(space.wrap(pw.c_pw_gid)),
-        space.wrap(rffi.charp2str(pw.c_pw_gecos)),
-        space.wrap(rffi.charp2str(pw.c_pw_dir)),
-        space.wrap(rffi.charp2str(pw.c_pw_shell)),
+        space.newtext(rffi.charp2str(pw.c_pw_name)),
+        space.newtext(rffi.charp2str(pw.c_pw_passwd)),
+        space.int(space.newint(pw.c_pw_uid)),
+        space.int(space.newint(pw.c_pw_gid)),
+        space.newtext(rffi.charp2str(pw.c_pw_gecos)),
+        space.newtext(rffi.charp2str(pw.c_pw_dir)),
+        space.newtext(rffi.charp2str(pw.c_pw_shell)),
         ])
     return space.call_function(w_passwd_struct, w_tuple)
 
@@ -102,7 +102,7 @@
         raise
     pw = c_getpwuid(uid)
     if not pw:
-        raise OperationError(space.w_KeyError, space.wrap(
+        raise OperationError(space.w_KeyError, space.newtext(
             "%s: %d" % (msg, widen(uid))))
     return make_struct_passwd(space, pw)
 

From pypy.commits at gmail.com  Thu Nov  3 11:46:03 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:46:03 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _file
Message-ID: <581b5bbb.12111c0a.1fc8d.7967@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88123:8a20e656b14d
Date: 2016-11-03 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/8a20e656b14d/

Log:	_file

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -57,7 +57,7 @@
         if self.space.sys.track_resources:
             w_repr = self.space.repr(self)
             str_repr = self.space.str_w(w_repr)
-            w_msg = self.space.wrap("WARNING: unclosed file: " + str_repr)
+            w_msg = self.space.newtext("WARNING: unclosed file: " + str_repr)
             self.space.resource_warning(w_msg, self.w_tb)
         #
         try:
@@ -159,7 +159,7 @@
 
     def direct_fdopen(self, fd, mode='r', buffering=-1):
         self.direct_close()
-        self.w_name = self.space.wrap('<fdopen>')
+        self.w_name = self.space.newtext('<fdopen>')
         self.check_mode_ok(mode)
         stream = streamio.fdopen_as_stream(fd, mode, buffering,
                                            signal_checker(self.space))
@@ -311,9 +311,9 @@
             if space.isinstance_w(w_data, space.w_unicode):
                 w_errors = w_encoding = None
                 if self.encoding:
-                    w_encoding = space.wrap(self.encoding)
+                    w_encoding = space.newtext(self.encoding)
                 if self.errors:
-                    w_errors = space.wrap(self.errors)
+                    w_errors = space.newtext(self.errors)
                 w_data = space.call_method(w_data, "encode",
                                            w_encoding, w_errors)
             data = space.charbuf_w(w_data)
@@ -349,9 +349,9 @@
     _exposed_method_names = []
 
     def _decl(class_scope, name, docstring,
-              wrapresult="space.wrap(result)"):
+              wrapresult="space.newtext(result)"):
         # hack hack to build a wrapper around the direct_xxx methods.
-        # The wrapper adds lock/unlock calls and a space.wrap() on
+        # The wrapper adds lock/unlock calls and a wraps
         # the result, conversion of stream errors to OperationErrors,
         # and has the specified docstring and unwrap_spec.
         direct_fn = class_scope['direct_' + name]
@@ -392,9 +392,11 @@
         class_scope['_exposed_method_names'].append(name)
 
 
-    _decl(locals(), "__init__", """Opens a file.""")
+    _decl(locals(), "__init__", """Opens a file.""",
+          wrapresult="space.w_None")
 
-    _decl(locals(), "__enter__", """__enter__() -> self.""")
+    _decl(locals(), "__enter__", """__enter__() -> self.""",
+          wrapresult="self")
 
     _decl(locals(), "close",
         """close() -> None or (perhaps) an integer.  Close the file.
@@ -402,7 +404,8 @@
 Sets data attribute .closed to True.  A closed file cannot be used for
 further I/O operations.  close() may be called more than once without
 error.  Some kinds of file objects (for example, opened by popen())
-may return an exit status upon closing.""")
+may return an exit status upon closing.""",
+          wrapresult="space.w_None")
         # NB. close() needs to use the stream lock to avoid double-closes or
         # close-while-another-thread-uses-it.
 
@@ -410,13 +413,16 @@
     _decl(locals(), "fileno",
         '''fileno() -> integer "file descriptor".
 
-This is needed for lower-level file interfaces, such os.read().''')
+This is needed for lower-level file interfaces, such os.read().''',
+          wrapresult="space.newint(result)")
 
     _decl(locals(), "flush",
-        """flush() -> None.  Flush the internal I/O buffer.""")
+        """flush() -> None.  Flush the internal I/O buffer.""",
+        wrapresult="space.w_None")
 
     _decl(locals(), "isatty",
-        """isatty() -> true or false.  True if the file is connected to a tty device.""")
+        """isatty() -> true or false.  True if the file is connected to a tty device.""",
+        wrapresult="space.newbool(result)")
 
     _decl(locals(), "next",
         """next() -> the next line in the file, or raise StopIteration""")
@@ -429,7 +435,8 @@
 may be returned, even if no size parameter was given.""")
 
     _decl(locals(), "readinto",
-        """readinto(buf) -> length.  Read into the given read-write buffer.""")
+        """readinto(buf) -> length.  Read into the given read-write buffer.""",
+        wrapresult="result")
 
     _decl(locals(), "readline",
         """readline([size]) -> next line from the file, as a string.
@@ -456,31 +463,37 @@
 seeking beyond the end of a file).  If the file is opened in text mode,
 only offsets returned by tell() are legal.  Use of other offsets causes
 undefined behavior.
-Note that not all file objects are seekable.""")
+Note that not all file objects are seekable.""",
+        wrapresult="space.w_None")
 
     _decl(locals(), "tell",
-        "tell() -> current file position, an integer (may be a long integer).")
+        "tell() -> current file position, an integer (may be a long integer).",
+        wrapresult="space.newint(result)")
 
     _decl(locals(), "truncate",
         """truncate([size]) -> None.  Truncate the file to at most size bytes.
 
-Size defaults to the current file position, as returned by tell().""")
+Size defaults to the current file position, as returned by tell().""",
+        wrapresult="space.w_None")
 
     _decl(locals(), "write",
         """write(str) -> None.  Write string str to file.
 
 Note that due to buffering, flush() or close() may be needed before
-the file on disk reflects the data written.""")
+the file on disk reflects the data written.""",
+        wrapresult="space.w_None")
 
     _decl(locals(), "__iter__",
         """Iterating over files, as in 'for line in f:', returns each line of
-the file one by one.""")
+the file one by one.""",
+        wrapresult="result")
 
     _decl(locals(), "xreadlines",
         """xreadlines() -> returns self.
 
 For backward compatibility. File objects now include the performance
-optimizations previously implemented in the xreadlines module.""")
+optimizations previously implemented in the xreadlines module.""",
+        wrapresult="result")
 
     def file__repr__(self):
         if self.stream is None:
@@ -523,7 +536,7 @@
                                 "writelines() argument must be a sequence of "
                                 "strings")
                 else:
-                    lines[i] = space.wrap(line)
+                    lines[i] = space.newtext(line)
         for w_line in lines:
             self.file_write(w_line)
 
@@ -534,17 +547,17 @@
 def descr_file__new__(space, w_subtype, __args__):
     file = space.allocate_instance(W_File, w_subtype)
     W_File.__init__(file, space)
-    return space.wrap(file)
+    return file
 
 @unwrap_spec(fd=int, mode=str, buffering=int)
 def descr_file_fdopen(space, w_subtype, fd, mode='r', buffering=-1):
     file = space.allocate_instance(W_File, w_subtype)
     W_File.__init__(file, space)
     file.file_fdopen(fd, mode, buffering)
-    return space.wrap(file)
+    return file
 
 def descr_file_closed(space, file):
-    return space.wrap(file.stream is None)
+    return space.newbool(file.stream is None)
 
 def descr_file_newlines(space, file):
     if file.stream:
@@ -554,22 +567,22 @@
     if newlines == 0:
         return space.w_None
     elif newlines == 1:
-        return space.wrap("\r")
+        return space.newtext("\r")
     elif newlines == 2:
-        return space.wrap("\n")
+        return space.newtext("\n")
     elif newlines == 4:
-        return space.wrap("\r\n")
+        return space.newtext("\r\n")
     result = []
     if newlines & 1:
-        result.append(space.wrap('\r'))
+        result.append(space.newtext('\r'))
     if newlines & 2:
-        result.append(space.wrap('\n'))
+        result.append(space.newtext('\n'))
     if newlines & 4:
-        result.append(space.wrap('\r\n'))
+        result.append(space.newtext('\r\n'))
     return space.newtuple(result[:])
 
 def descr_file_softspace(space, file):
-    return space.wrap(file.softspace)
+    return space.newint(file.softspace)
 
 def descr_file_setsoftspace(space, file, w_newvalue):
     file.softspace = space.int_w(w_newvalue)

From pypy.commits at gmail.com  Thu Nov  3 11:46:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:46:05 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _multiprocessing
Message-ID: <581b5bbd.876ec20a.12fc1.e21e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88124:6301ac648043
Date: 2016-11-03 15:01 +0100
http://bitbucket.org/pypy/pypy/changeset/6301ac648043/

Log:	_multiprocessing

diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -22,18 +22,18 @@
     def init(self, space):
         w_builtins = space.getbuiltinmodule('__builtin__')
         w_module = space.call_method(
-            w_builtins, '__import__', space.wrap("multiprocessing"))
-        self.w_BufferTooShort = space.getattr(w_module, space.wrap("BufferTooShort"))
+            w_builtins, '__import__', space.newtext("multiprocessing"))
+        self.w_BufferTooShort = space.getattr(w_module, space.newtext("BufferTooShort"))
 
         self.w_picklemodule = space.call_method(
-            w_builtins, '__import__', space.wrap("pickle"))
+            w_builtins, '__import__', space.newtext("pickle"))
 
 def BufferTooShort(space, w_data):
     w_BufferTooShort = space.fromcache(State).w_BufferTooShort
     return OperationError(w_BufferTooShort, w_data)
 
 def w_handle(space, handle):
-    return space.wrap(rffi.cast(rffi.INTPTR_T, handle))
+    return space.newint(rffi.cast(rffi.INTPTR_T, handle))
 
 
 class W_BaseConnection(W_Root):
@@ -80,7 +80,7 @@
 
     def _repr(self, space, handle):
         conn_type = ["read-only", "write-only", "read-write"][self.flags - 1]
-        return space.wrap("<%s %s, handle %d>" % (
+        return space.newtext("<%s %s, handle %d>" % (
                 conn_type, space.type(self).getname(space), handle))
 
     def descr_repr(self, space):
@@ -145,14 +145,14 @@
             if newbuf:
                 rffi.free_charp(newbuf)
 
-        return space.wrap(res)
+        return space.newint(res)
 
     def send(self, space, w_obj):
         self._check_writable(space)
 
         w_picklemodule = space.fromcache(State).w_picklemodule
         w_protocol = space.getattr(
-            w_picklemodule, space.wrap("HIGHEST_PROTOCOL"))
+            w_picklemodule, space.newtext("HIGHEST_PROTOCOL"))
         w_pickled = space.call_method(
             w_picklemodule, "dumps", w_obj, w_protocol)
 
@@ -254,13 +254,13 @@
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
         W_FileConnection.__init__(self, space, fd, flags)
-        return space.wrap(self)
+        return self
 
     def descr_repr(self, space):
         return self._repr(space, self.fd)
 
     def fileno(self, space):
-        return space.wrap(self.fd)
+        return space.newint(self.fd)
 
     def is_valid(self):
         return self.fd != self.INVALID_HANDLE_VALUE
@@ -379,7 +379,7 @@
 
         self = space.allocate_instance(W_PipeConnection, w_subtype)
         W_PipeConnection.__init__(self, space, handle, flags)
-        return space.wrap(self)
+        return self
 
     def descr_repr(self, space):
         return self._repr(space, rffi.cast(rffi.INTPTR_T, self.handle))
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -445,27 +445,27 @@
         return w_handle(space, self.handle)
 
     def get_count(self, space):
-        return space.wrap(self.count)
+        return space.newint(self.count)
 
     def _ismine(self):
         return self.count > 0 and rthread.get_ident() == self.last_tid
 
     def is_mine(self, space):
-        return space.wrap(self._ismine())
+        return space.newbool(self._ismine())
 
     def is_zero(self, space):
         try:
             res = semlock_iszero(self, space)
         except OSError as e:
             raise wrap_oserror(space, e)
-        return space.wrap(res)
+        return space.newbool(res)
 
     def get_value(self, space):
         try:
             val = semlock_getvalue(self, space)
         except OSError as e:
             raise wrap_oserror(space, e)
-        return space.wrap(val)
+        return space.newint(val)
 
     @unwrap_spec(block=bool)
     def acquire(self, space, block=True, w_timeout=None):
@@ -510,7 +510,7 @@
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
         self.__init__(space, handle_w(space, w_handle), kind, maxvalue)
-        return space.wrap(self)
+        return self
 
     def enter(self, space):
         return self.acquire(space, w_timeout=space.w_None)
@@ -537,7 +537,7 @@
     self = space.allocate_instance(W_SemLock, w_subtype)
     self.__init__(space, handle, kind, maxvalue)
 
-    return space.wrap(self)
+    return self
 
 W_SemLock.typedef = TypeDef(
     "SemLock",

From pypy.commits at gmail.com  Thu Nov  3 11:46:07 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:46:07 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _ssl
Message-ID: <581b5bbf.838e1c0a.944b4.1121@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88125:86e143f3511c
Date: 2016-11-03 16:09 +0100
http://bitbucket.org/pypy/pypy/changeset/86e143f3511c/

Log:	_ssl

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -127,13 +127,13 @@
     w_exception_class = w_errtype or get_exception_class(space, 'w_sslerror')
     if errno or errcode:
         w_exception = space.call_function(w_exception_class,
-                                          space.wrap(errno), space.wrap(msg))
+                                          space.newint(errno), space.newtext(msg))
     else:
-        w_exception = space.call_function(w_exception_class, space.wrap(msg))
-    space.setattr(w_exception, space.wrap("reason"),
-                  space.wrap(reason_str) if reason_str else space.w_None)
-    space.setattr(w_exception, space.wrap("library"),
-                  space.wrap(lib_str) if lib_str else space.w_None)
+        w_exception = space.call_function(w_exception_class, space.newtext(msg))
+    space.setattr(w_exception, space.newtext("reason"),
+                  space.newtext_or_none(reason_str))
+    space.setattr(w_exception, space.newtext("library"),
+                  space.newtext_or_none(lib_str))
     return OperationError(w_exception_class, w_exception)
 
 class SSLNpnProtocols(object):
@@ -244,7 +244,7 @@
         on some platforms before using the ssl() function."""
 
         res = libssl_RAND_status()
-        return space.wrap(res)
+        return space.newint(res)
 
     if HAVE_OPENSSL_RAND_EGD:
         @unwrap_spec(path=str)
@@ -260,7 +260,7 @@
                 raise ssl_error(space,
                                 "EGD connection failed or EGD did not return "
                                 "enough data to seed the PRNG")
-            return space.wrap(bytes)
+            return space.newint(bytes)
     else:
         # Dummy func for platforms missing RAND_egd(). Most likely LibreSSL.
         @unwrap_spec(path=str)
@@ -298,7 +298,7 @@
         if not space.is_none(w_hostname):
             if space.isinstance_w(w_hostname, space.w_unicode):
                 w_hostname = space.call_method(w_hostname, "encode",
-                                               space.wrap("idna"))
+                                               space.newtext("idna"))
             libssl_SSL_set_tlsext_host_name(
                 self.ssl, space.bytes_w(w_hostname))
 
@@ -377,7 +377,7 @@
                 break
 
         if num_bytes > 0:
-            return space.wrap(num_bytes)
+            return space.newint(num_bytes)
         else:
             raise _ssl_seterror(space, self, num_bytes)
 
@@ -389,7 +389,7 @@
         count = libssl_SSL_pending(self.ssl)
         if count < 0:
             raise _ssl_seterror(space, self, count)
-        return space.wrap(count)
+        return space.newint(count)
 
     @unwrap_spec(num_bytes=int)
     def read(self, space, num_bytes, w_buffer=None):
@@ -406,7 +406,7 @@
                                 "Underlying socket too large for select().")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN:
-                    return space.wrap('')
+                    return space.newbytes('')
                 raise ssl_error(space,
                                 "Socket closed without SSL shutdown handshake")
 
@@ -434,9 +434,9 @@
                 elif (err == SSL_ERROR_ZERO_RETURN and
                         libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
                     if rwbuffer:
-                        return space.wrap(0)
+                        return space.newint(0)
                     else:
-                        return space.wrap("")
+                        return space.newbytes("")
                 else:
                     sockstate = SOCKET_OPERATION_OK
 
@@ -457,9 +457,9 @@
 
         if rwbuffer:
             rwbuffer.setslice(0, result)
-            return space.wrap(count)
+            return space.newint(count)
         else:
-            return space.wrap(result)
+            return space.newbytes(result)
 
     def _refresh_nonblocking(self, space):
         # just in case the blocking state of the socket has been changed
@@ -575,10 +575,10 @@
             return space.w_None
 
         name = libssl_SSL_CIPHER_get_name(current)
-        w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None
+        w_name = space.newtext(rffi.charp2str(name)) if name else space.w_None
 
         proto = libssl_SSL_CIPHER_get_version(current)
-        w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None
+        w_proto = space.newtext(rffi.charp2str(proto)) if proto else space.w_None
 
         bits = libssl_SSL_CIPHER_get_bits(current,
                                           lltype.nullptr(rffi.INTP.TO))
@@ -623,7 +623,7 @@
                 libssl_SSL_get0_next_proto_negotiated(self.ssl,
                                                       out_ptr, len_ptr)
                 if out_ptr[0]:
-                    return space.wrap(
+                    return space.newtext(
                         rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
 
     def selected_alpn_protocol(self, space):
@@ -635,7 +635,7 @@
                 libssl_SSL_get0_alpn_selected(self.ssl,
                                               out_ptr, len_ptr)
                 if out_ptr[0]:
-                    return space.wrap(
+                    return space.newtext(
                         rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
 
     def compression_w(self, space):
@@ -650,7 +650,7 @@
         short_name = libssl_COMP_get_name(comp_method)
         if not short_name:
             return space.w_None
-        return space.wrap(rffi.charp2str(short_name))
+        return space.newtext(rffi.charp2str(short_name))
 
     def version_w(self, space):
         if not self.ssl:
@@ -658,7 +658,7 @@
         version = libssl_SSL_get_version(self.ssl)
         if not version:
             return space.w_None
-        return space.wrap(rffi.charp2str(version))
+        return space.newtext(rffi.charp2str(version))
 
     def tls_unique_cb_w(self, space):
         """Returns the 'tls-unique' channel binding data, as defined by RFC 5929.
@@ -728,14 +728,14 @@
 
     w_peer = _create_tuple_for_X509_NAME(
         space, libssl_X509_get_subject_name(certificate))
-    space.setitem(w_retval, space.wrap("subject"), w_peer)
+    space.setitem(w_retval, space.newtext("subject"), w_peer)
 
     w_issuer = _create_tuple_for_X509_NAME(
         space, libssl_X509_get_issuer_name(certificate))
-    space.setitem(w_retval, space.wrap("issuer"), w_issuer)
+    space.setitem(w_retval, space.newtext("issuer"), w_issuer)
 
-    space.setitem(w_retval, space.wrap("version"),
-                  space.wrap(libssl_X509_get_version(certificate) + 1))
+    space.setitem(w_retval, space.newtext("version"),
+                  space.newint(libssl_X509_get_version(certificate) + 1))
 
     biobuf = libssl_BIO_new(libssl_BIO_s_mem())
     try:
@@ -749,8 +749,8 @@
             if length < 0:
                 raise _ssl_seterror(space, None, length)
 
-            w_serial = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("serialNumber"), w_serial)
+            w_serial = space.newtext(rffi.charpsize2str(buf, length))
+        space.setitem(w_retval, space.newtext("serialNumber"), w_serial)
 
         libssl_BIO_reset(biobuf)
         notBefore = libssl_X509_get_notBefore(certificate)
@@ -759,8 +759,8 @@
             length = libssl_BIO_gets(biobuf, buf, 99)
             if length < 0:
                 raise _ssl_seterror(space, None, length)
-            w_date = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("notBefore"), w_date)
+            w_date = space.newtext(rffi.charpsize2str(buf, length))
+        space.setitem(w_retval, space.newtext("notBefore"), w_date)
 
         libssl_BIO_reset(biobuf)
         notAfter = libssl_X509_get_notAfter(certificate)
@@ -769,28 +769,28 @@
             length = libssl_BIO_gets(biobuf, buf, 99)
             if length < 0:
                 raise _ssl_seterror(space, None, length)
-            w_date = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("notAfter"), w_date)
+            w_date = space.newtext(rffi.charpsize2str(buf, length))
+        space.setitem(w_retval, space.newtext("notAfter"), w_date)
     finally:
         libssl_BIO_free(biobuf)
 
     # Now look for subjectAltName
     w_alt_names = _get_peer_alt_names(space, certificate)
     if w_alt_names is not space.w_None:
-        space.setitem(w_retval, space.wrap("subjectAltName"), w_alt_names)
+        space.setitem(w_retval, space.newtext("subjectAltName"), w_alt_names)
 
     # Authority Information Access: OCSP URIs
     w_ocsp = _get_aia_uri(space, certificate, NID_ad_OCSP)
     if not space.is_none(w_ocsp):
-        space.setitem(w_retval, space.wrap("OCSP"), w_ocsp)
+        space.setitem(w_retval, space.newtext("OCSP"), w_ocsp)
     w_issuers = _get_aia_uri(space, certificate, NID_ad_ca_issuers)
     if not space.is_none(w_issuers):
-        space.setitem(w_retval, space.wrap("caIssuers"), w_issuers)
+        space.setitem(w_retval, space.newtext("caIssuers"), w_issuers)
 
     # CDP (CRL distribution points)
     w_cdp = _get_crl_dp(space, certificate)
     if not space.is_none(w_cdp):
-        space.setitem(w_retval, space.wrap("crlDistributionPoints"), w_cdp)
+        space.setitem(w_retval, space.newtext("crlDistributionPoints"), w_cdp)
 
     return w_retval
 
@@ -877,18 +877,18 @@
                         # attributes
                         dirname = libssl_pypy_GENERAL_NAME_dirn(name)
                         w_t = space.newtuple([
-                            space.wrap("DirName"),
+                            space.newtext("DirName"),
                             _create_tuple_for_X509_NAME(space, dirname)
                             ])
                     elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
                         # GENERAL_NAME_print() doesn't handle NULL bytes in
                         # ASN1_string correctly, CVE-2013-4238
                         if gntype == GEN_EMAIL:
-                            v = space.wrap("email")
+                            v = space.newtext("email")
                         elif gntype == GEN_DNS:
-                            v = space.wrap("DNS")
+                            v = space.newtext("DNS")
                         elif gntype == GEN_URI:
-                            v = space.wrap("URI")
+                            v = space.newtext("URI")
                         else:
                             assert False
                         as_ = libssl_pypy_GENERAL_NAME_dirn(name)
@@ -896,12 +896,12 @@
                         buf = libssl_ASN1_STRING_data(as_)
                         length = libssl_ASN1_STRING_length(as_)
                         w_t = space.newtuple([
-                            v, space.wrap(rffi.charpsize2str(buf, length))])
+                            v, space.newtext(rffi.charpsize2str(buf, length))])
                     else:
                         # for everything else, we use the OpenSSL print form
                         if gntype not in (GEN_OTHERNAME, GEN_X400, GEN_EDIPARTY,
                                           GEN_IPADD, GEN_RID):
-                            space.warn(space.wrap("Unknown general name type"),
+                            space.warn(space.newtext("Unknown general name type"),
                                        space.w_RuntimeWarning)
                         libssl_BIO_reset(biobuf)
                         libssl_GENERAL_NAME_print(biobuf, name)
@@ -912,8 +912,8 @@
 
                             v = rffi.charpsize2str(buf, length)
                         v1, v2 = v.split(':', 1)
-                        w_t = space.newtuple([space.wrap(v1),
-                                              space.wrap(v2)])
+                        w_t = space.newtuple([space.newtext(v1),
+                                              space.newtext(v2)])
 
                     alt_names_w.append(w_t)
             finally:
@@ -932,7 +932,7 @@
         length = libssl_OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
         if length < 0:
             raise _ssl_seterror(space, None, 0)
-        w_name = space.wrap(rffi.charpsize2str(buf, length))
+        w_name = space.newtext(rffi.charpsize2str(buf, length))
 
     with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as buf_ptr:
         length = libssl_ASN1_STRING_to_UTF8(buf_ptr, value)
@@ -940,7 +940,7 @@
             raise _ssl_seterror(space, None, 0)
         try:
             w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length))
-            w_value = space.call_method(w_value, "decode", space.wrap("utf-8"))
+            w_value = space.call_method(w_value, "decode", space.newtext("utf-8"))
         finally:
             libssl_OPENSSL_free(buf_ptr[0])
 
@@ -967,7 +967,7 @@
             uri = libssl_pypy_GENERAL_NAME_uri(name)
             length = intmask(uri.c_length)
             s_uri = rffi.charpsize2str(uri.c_data, length)
-            result_w.append(space.wrap(s_uri))
+            result_w.append(space.newtext(s_uri))
         return space.newtuple(result_w[:])
     finally:
         libssl_AUTHORITY_INFO_ACCESS_free(info)
@@ -995,7 +995,7 @@
                 uri = libssl_pypy_GENERAL_NAME_uri(name)
                 length = intmask(uri.c_length)
                 s_uri = rffi.charpsize2str(uri.c_data, length)
-                cdp_w.append(space.wrap(s_uri))
+                cdp_w.append(space.newtext(s_uri))
     finally:
         if OPENSSL_VERSION_NUMBER < 0x10001000:
             libssl_sk_DIST_POINT_free(dps)
@@ -1120,10 +1120,10 @@
                      errcode=errcode)
 
 def SSLError_descr_str(space, w_exc):
-    w_strerror = space.getattr(w_exc, space.wrap("strerror"))
+    w_strerror = space.getattr(w_exc, space.newtext("strerror"))
     if not space.is_none(w_strerror):
         return w_strerror
-    return space.str(space.getattr(w_exc, space.wrap("args")))
+    return space.str(space.getattr(w_exc, space.newtext("args")))
 
 
 class Cache:
@@ -1131,8 +1131,8 @@
         w_socketerror = interp_socket.get_error(space, "error")
         self.w_sslerror = space.new_exception_class(
             "_ssl.SSLError", w_socketerror)
-        space.setattr(self.w_sslerror, space.wrap('__str__'),
-                      space.wrap(interp2app(SSLError_descr_str)))
+        space.setattr(self.w_sslerror, space.newtext('__str__'),
+                      interp2app(SSLError_descr_str).__spacebind__(space))
         self.w_sslzeroreturnerror = space.new_exception_class(
             "_ssl.SSLZeroReturnError", self.w_sslerror)
         self.w_sslwantreaderror = space.new_exception_class(
@@ -1246,7 +1246,7 @@
             w_servername = space.newbytes(rffi.charp2str(servername))
             try:
                 w_servername_idna = space.call_method(
-                    w_servername, 'decode', space.wrap('idna'))
+                    w_servername, 'decode', space.newtext('idna'))
             except OperationError as e:
                 e.write_unraisable(space, "undecodable server name")
                 ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
@@ -1344,7 +1344,7 @@
     def session_stats_w(self, space):
         w_stats = space.newdict()
         for name, ssl_func in SSL_CTX_STATS:
-            w_value = space.wrap(ssl_func(self.ctx))
+            w_value = space.newint(ssl_func(self.ctx))
             space.setitem_str(w_stats, name, w_value)
         return w_stats
 
@@ -1400,7 +1400,7 @@
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
         param = libssl_X509_STORE_get0_param(store)
         flags = libssl_X509_VERIFY_PARAM_get_flags(param)
-        return space.wrap(flags)
+        return space.newint(flags)
 
     def descr_set_verify_flags(self, space, w_obj):
         new_flags = space.int_w(w_obj)
@@ -1642,11 +1642,11 @@
                 pass
         w_result = space.newdict()
         space.setitem(w_result,
-                      space.wrap('x509'), space.wrap(x509))
+                      space.newtext('x509'), space.newint(x509))
         space.setitem(w_result,
-                      space.wrap('x509_ca'), space.wrap(x509_ca))
+                      space.newtext('x509_ca'), space.newint(x509_ca))
         space.setitem(w_result,
-                      space.wrap('crl'), space.wrap(crl))
+                      space.newtext('crl'), space.newint(crl))
         return w_result
 
     @unwrap_spec(protos='bufferstr')
@@ -1759,12 +1759,12 @@
         if buflen < 0:
             raise _ssl_seterror(space, None, 0)
         if buflen:
-            w_buf = space.wrap(buf.str(buflen))
+            w_buf = space.newtext(buf.str(buflen))
         else:
             w_buf = space.w_None
-    w_sn = space.wrap(rffi.charp2str(libssl_OBJ_nid2sn(nid)))
-    w_ln = space.wrap(rffi.charp2str(libssl_OBJ_nid2ln(nid)))
-    return space.newtuple([space.wrap(nid), w_sn, w_ln, w_buf])
+    w_sn = space.newtext(rffi.charp2str(libssl_OBJ_nid2sn(nid)))
+    w_ln = space.newtext(rffi.charp2str(libssl_OBJ_nid2ln(nid)))
+    return space.newtuple([space.newint(nid), w_sn, w_ln, w_buf])
 
 
 @unwrap_spec(txt=str, name=bool)

From pypy.commits at gmail.com  Thu Nov  3 11:58:18 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:58:18 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: zipimport
Message-ID: <581b5e9a.c1341c0a.6e1f3.1a9a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88127:a3f02433a915
Date: 2016-11-03 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/a3f02433a915/

Log:	zipimport

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -51,21 +51,21 @@
         try:
             w_zipimporter = self.cache[name]
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap(name))
+            raise OperationError(space.w_KeyError, space.newtext(name))
         assert isinstance(w_zipimporter, W_ZipImporter)
         w = space.wrap
         w_d = space.newdict()
         for key, info in w_zipimporter.zip_file.NameToInfo.iteritems():
             if ZIPSEP != os.path.sep:
                 key = key.replace(ZIPSEP, os.path.sep)
-            space.setitem(w_d, w(key), space.newtuple([
+            space.setitem(w_d, space.newtext(key), space.newtuple([
                 w(info.filename), w(info.compress_type), w(info.compress_size),
                 w(info.file_size), w(info.file_offset), w(info.dostime),
                 w(info.dosdate), w(info.CRC)]))
         return w_d
 
     def keys(self, space):
-        return space.newlist([space.wrap(s)
+        return space.newlist([space.newtext(s)
                               for s in self.cache.keys()])
 
     def values(self, space):
@@ -75,7 +75,7 @@
 
     def items(self, space):
         w = space.wrap
-        items_w = [space.newtuple([w(key), self.getitem(space, key)])
+        items_w = [space.newtuple([space.newtext(key), self.getitem(space, key)])
                    for key in self.cache.keys()]
         return space.newlist(items_w)
 
@@ -126,8 +126,8 @@
 
     def getprefix(self, space):
         if ZIPSEP == os.path.sep:
-            return space.wrap(self.prefix)
-        return space.wrap(self.prefix.replace(ZIPSEP, os.path.sep))
+            return space.newtext(self.prefix)
+        return space.newtext(self.prefix.replace(ZIPSEP, os.path.sep))
 
     def _find_relative_path(self, filename):
         if filename.startswith(self.filename):
@@ -146,16 +146,15 @@
 
     def import_py_file(self, space, modname, filename, buf, pkgpath):
         w = space.wrap
-        w_mod = w(Module(space, w(modname)))
+        w_mod = Module(space, space.newtext(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
-        space.setattr(w_mod, w('__loader__'), space.wrap(self))
+        space.setattr(w_mod, space.newtext('__loader__'), self)
         importing._prepare_module(space, w_mod, real_name, pkgpath)
         co_filename = self.make_co_filename(filename)
         code_w = importing.parse_source_module(space, co_filename, buf)
-        return importing.exec_code_module(space, w_mod, code_w, w(modname))
+        return importing.exec_code_module(space, w_mod, code_w, space.newtext(modname))
 
     def _parse_mtime(self, space, filename):
-        w = space.wrap
         try:
             info = self.zip_file.NameToInfo[filename]
             t = info.date_time
@@ -163,13 +162,15 @@
             return 0
         else:
             w_mktime = space.getattr(space.getbuiltinmodule('time'),
-                                     w('mktime'))
+                                     space.newtext('mktime'))
             # XXX this is incredible fishing around module limitations
             #     in order to compare timestamps of .py and .pyc files
             # we need time.mktime support on rpython level
+            w = space.newint
             all = [w(t[0]), w(t[1]), w(t[2]), w(t[3]), w(t[4]),
                    w(t[5]), w(0), w(1), w(-1)]
-            mtime = int(space.float_w(space.call_function(w_mktime, space.newtuple(all))))
+            mtime = int(space.float_w(space.call_function(
+                w_mktime, space.newtuple(all))))
             return mtime
 
     def check_newer_pyfile(self, space, filename, timestamp):
@@ -194,17 +195,16 @@
         return True
 
     def import_pyc_file(self, space, modname, filename, buf, pkgpath):
-        w = space.wrap
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
             return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
-        w_mod = w(Module(space, w(modname)))
+        w_mod = Module(space, space.newtext(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
-        space.setattr(w_mod, w('__loader__'), space.wrap(self))
+        space.setattr(w_mod, space.newtext('__loader__'), self)
         importing._prepare_module(space, w_mod, real_name, pkgpath)
-        w_result = importing.load_compiled_module(space, w(modname), w_mod,
+        w_result = importing.load_compiled_module(space, space.newtext(modname), w_mod,
                                                 filename, magic, timestamp,
                                                 buf)
         return w_result
@@ -223,7 +223,7 @@
         filename = self.make_filename(fullname)
         for _, _, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(self)
+                return self
 
     def make_filename(self, fullname):
         startpos = fullname.rfind('.') + 1 # 0 when not found
@@ -245,7 +245,6 @@
 
     @unwrap_spec(fullname=str)
     def load_module(self, space, fullname):
-        w = space.wrap
         filename = self.make_filename(fullname)
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
@@ -274,17 +273,16 @@
                                                    buf, pkgpath)
                 except:
                     w_mods = space.sys.get('modules')
-                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    space.call_method(w_mods, 'pop', space.newtext(fullname), space.w_None)
                     raise
         raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename=str)
     def get_data(self, space, filename):
         filename = self._find_relative_path(filename)
-        w = space.wrap
         try:
             data = self.zip_file.read(filename)
-            return w(data)
+            return space.newbytes(data)
         except (KeyError, OSError, BadZipfile):
             raise oefmt(space.w_IOError, "Error reading file")
         except RZlibError as e:
@@ -305,13 +303,13 @@
                     if not self.can_use_pyc(space, filename + ext,
                                             magic, timestamp):
                         continue
-                    code_w = importing.read_compiled_module(
+                    w_code = importing.read_compiled_module(
                         space, filename + ext, source[8:])
                 else:
                     co_filename = self.make_co_filename(filename+ext)
-                    code_w = importing.parse_source_module(
+                    w_code = importing.parse_source_module(
                         space, co_filename, source)
-                return space.wrap(code_w)
+                return w_code
         raise oefmt(get_error(space),
                     "Cannot find source or code for %s in %s",
                     filename, self.name)
@@ -337,8 +335,8 @@
         filename = self.make_filename(fullname)
         for _, is_package, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(self.filename + os.path.sep +
-                                  self.corr_zname(filename + ext))
+                return space.newtext(self.filename + os.path.sep +
+                                     self.corr_zname(filename + ext))
         raise oefmt(get_error(space),
                     "Cannot find module %s in %s", filename, self.name)
 
@@ -347,13 +345,13 @@
         filename = self.make_filename(fullname)
         for _, is_package, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(is_package)
+                return space.newbool(is_package)
         raise oefmt(get_error(space),
                     "Cannot find module %s in %s", filename, self.name)
 
     def getarchive(self, space):
         space = self.space
-        return space.wrap(self.filename)
+        return space.newtext(self.filename)
 
 @unwrap_spec(name='str0')
 def descr_new_zipimporter(space, w_type, name):
@@ -398,7 +396,7 @@
         prefix = prefix[1:]
     if prefix and not prefix.endswith(ZIPSEP) and not prefix.endswith(os.path.sep):
         prefix += ZIPSEP
-    w_result = space.wrap(W_ZipImporter(space, name, filename, zip_file, prefix))
+    w_result = W_ZipImporter(space, name, filename, zip_file, prefix)
     zip_cache.set(filename, w_result)
     return w_result
 

From pypy.commits at gmail.com  Thu Nov  3 11:58:20 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 Nov 2016 08:58:20 -0700 (PDT)
Subject: [pypy-commit] pypy space-newtext: _lsprof
Message-ID: <581b5e9c.231cc20a.37924.fd43@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88128:58b765d023e6
Date: 2016-11-03 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/58b765d023e6/

Log:	_lsprof

diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -54,7 +54,7 @@
             calls_repr = "None"
         else:
             calls_repr = space.str_w(space.repr(self.w_calls))
-        return space.wrap('("%s", %d, %d, %f, %f, %s)' % (
+        return space.newtext('("%s", %d, %d, %f, %f, %s)' % (
             frame_repr, self.callcount, self.reccallcount,
             self.tt, self.it, calls_repr))
 
@@ -82,7 +82,7 @@
 
     def repr(self, space):
         frame_repr = space.str_w(space.repr(self.frame))
-        return space.wrap('("%s", %d, %d, %f, %f)' % (
+        return space.newtext('("%s", %d, %d, %f, %f)' % (
             frame_repr, self.callcount, self.reccallcount, self.tt, self.it))
 
     def get_code(self, space):
@@ -119,7 +119,7 @@
                                 self.callcount, self.recursivecallcount,
                                 factor * float(self.ll_tt),
                                 factor * float(self.ll_it))
-        return space.wrap(w_sse)
+        return w_sse
 
     def _stop(self, tt, it):
         if not we_are_translated():
@@ -148,7 +148,7 @@
                             self.recursivecallcount,
                             factor * float(self.ll_tt),
                             factor * float(self.ll_it), w_sublist)
-        return space.wrap(w_se)
+        return w_se
 
     @jit.elidable
     def _get_or_make_subentry(self, entry, make=True):
@@ -243,7 +243,7 @@
                 s = create_spec_for_object(space, self.w_type)
             else:
                 s = create_spec_for_method(space, self.w_func, self.w_type)
-            self.w_string = space.wrap(s)
+            self.w_string = space.newtext(s)
         return self.w_string
 
 W_DelayedBuiltinStr.typedef = TypeDef(
@@ -328,7 +328,7 @@
         self.total_timestamp -= read_timestamp()
         # set profiler hook
         c_setup_profiling()
-        space.getexecutioncontext().setllprofile(lsprof_call, space.wrap(self))
+        space.getexecutioncontext().setllprofile(lsprof_call, self)
 
     @jit.elidable
     def _get_or_make_entry(self, f_code, make=True):
@@ -435,7 +435,7 @@
                       subcalls=True, builtins=True):
     p = space.allocate_instance(W_Profiler, w_type)
     p.__init__(space, w_callable, time_unit, subcalls, builtins)
-    return space.wrap(p)
+    return p
 
 W_Profiler.typedef = TypeDef(
     '_lsprof.Profiler',

From pypy.commits at gmail.com  Thu Nov  3 13:38:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 03 Nov 2016 10:38:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: reversed({})
Message-ID: <581b7603.88abc20a.2a66b.26cb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88130:a3cce435fa17
Date: 2016-11-03 18:37 +0100
http://bitbucket.org/pypy/pypy/changeset/a3cce435fa17/

Log:	reversed({})

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -328,6 +328,9 @@
         if w_reversed_descr is not None:
             w_reversed = space.get(w_reversed_descr, w_sequence)
             return space.call_function(w_reversed)
+        if not space.issequence_w(w_sequence):
+            raise oefmt(space.w_TypeError,
+                        "argument to reversed() must be a sequence")
         self = space.allocate_instance(W_ReversedIterator, w_subtype)
         self.__init__(space, w_sequence)
         return space.wrap(self)
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -527,6 +527,9 @@
         assert list(reversed(list(reversed("hello")))) == ['h','e','l','l','o']
         raises(TypeError, reversed, reversed("hello"))
 
+    def test_reversed_nonsequence(self):
+        raises(TypeError, reversed, {})
+
     def test_reversed_length_hint(self):
         lst = [1, 2, 3]
         r = reversed(lst)

From pypy.commits at gmail.com  Thu Nov  3 13:38:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 03 Nov 2016 10:38:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Comment
Message-ID: <581b7601.88abc20a.2a66b.26c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88129:406ae24f4421
Date: 2016-11-03 18:30 +0100
http://bitbucket.org/pypy/pypy/changeset/406ae24f4421/

Log:	Comment

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1497,7 +1497,7 @@
             mtime = mtime_s + (mtime_ns / 1e9)
             func(arg, (atime, mtime))
     except OSError as e:
-        # see comment above
+        # see comment above: don't use wrap_oserror2()
         raise wrap_oserror(space, e)
 
 @specialize.argtype(1)

From pypy.commits at gmail.com  Thu Nov  3 14:07:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 03 Nov 2016 11:07:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <581b7cdd.cf9a1c0a.e0d2c.125b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88131:8b664111697d
Date: 2016-11-03 19:06 +0100
http://bitbucket.org/pypy/pypy/changeset/8b664111697d/

Log:	fix test

diff --git a/lib-python/3/test/test_bytes.py b/lib-python/3/test/test_bytes.py
--- a/lib-python/3/test/test_bytes.py
+++ b/lib-python/3/test/test_bytes.py
@@ -1006,9 +1006,13 @@
     def test_del_expand(self):
         # Reducing the size should not expand the buffer (issue #23985)
         b = bytearray(10)
-        size = sys.getsizeof(b)
-        del b[:1]
-        self.assertLessEqual(sys.getsizeof(b), size)
+        try:
+            size = sys.getsizeof(b)
+        except TypeError:
+            pass            # e.g. on pypy
+        else:
+            del b[:1]
+            self.assertLessEqual(sys.getsizeof(b), size)
 
     def test_extended_set_del_slice(self):
         indices = (0, None, 1, 3, 19, 300, 1<<333, -1, -2, -31, -300)

From pypy.commits at gmail.com  Thu Nov  3 14:30:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 03 Nov 2016 11:30:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for -A
Message-ID: <581b8256.85951c0a.8e96b.1d7a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88132:cb0741c5a10c
Date: 2016-11-03 19:22 +0100
http://bitbucket.org/pypy/pypy/changeset/cb0741c5a10c/

Log:	Fix for -A

diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -140,8 +140,9 @@
     def test_format_wrongtype(self):
         for int_format in '%d', '%o', '%x':
             exc_info = raises(TypeError, int_format.__mod__, '123')
-            expected = int_format + ' format: a number is required, not str'
-            assert str(exc_info.value) == expected
+            expected1 = int_format + ' format: a number is required, not str'
+            expected2 = int_format + ' format: an integer is required, not str'
+            assert str(exc_info.value) in (expected1, expected2)
         raises(TypeError, "None % 'abc'") # __rmod__
 
     def test_split(self):

From pypy.commits at gmail.com  Thu Nov  3 14:30:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 03 Nov 2016 11:30:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Two tests (failing so far)
Message-ID: <581b8258.4d881c0a.45824.1f83@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88133:c52c751b137d
Date: 2016-11-03 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/c52c751b137d/

Log:	Two tests (failing so far)

diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -545,3 +545,6 @@
         raises(TypeError, 'bytearray(b"%b") % "hello world"')
         assert bytearray(b'%b %b') % (b'a', bytearray(b'f f e')) == b'a f f e'
         """
+
+    def test_format_bytes(self):
+        assert bytearray(b'<%s>') % b'abc' == b'<abc>'
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -145,6 +145,9 @@
             assert str(exc_info.value) in (expected1, expected2)
         raises(TypeError, "None % 'abc'") # __rmod__
 
+    def test_format_bytes(self):
+        assert b'<%s>' % b'abc' == b'<abc>'
+
     def test_split(self):
         assert b"".split() == []
         assert b"".split(b'x') == [b'']

From pypy.commits at gmail.com  Thu Nov  3 19:52:03 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 03 Nov 2016 16:52:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement
 bytes.decode(errors='backslashreplace')
Message-ID: <581bcda3.41d71c0a.3b445.88d1@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88134:647ced05d718
Date: 2016-11-03 22:31 +0100
http://bitbucket.org/pypy/pypy/changeset/647ced05d718/

Log:	Implement bytes.decode(errors='backslashreplace')

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,8 +1,9 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
-    code_to_unichr, MAXUNICODE, raw_unicode_escape_helper)
+    code_to_unichr, MAXUNICODE,
+    raw_unicode_escape_helper, raw_unicode_escape_helper)
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -260,14 +261,15 @@
             builder.append(str(code))
             builder.append(";")
             pos += 1
-        return space.newtuple([space.wrap(builder.build()), w_end])
+        return space.newtuple([space.newbytes(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
 
 def backslashreplace_errors(space, w_exc):
     check_exception(space, w_exc)
-    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+    if (space.isinstance_w(w_exc, space.w_UnicodeEncodeError) or
+        space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
         obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
@@ -278,6 +280,18 @@
             oc = ord(obj[pos])
             raw_unicode_escape_helper(builder, oc)
             pos += 1
+        return space.newtuple([space.newbytes(builder.build()), w_end])
+    elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
+        obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        builder = UnicodeBuilder()
+        pos = start
+        while pos < end:
+            oc = ord(obj[pos])
+            runicode.raw_unicode_escape_helper_unicode(builder, oc)
+            pos += 1
         return space.newtuple([space.wrap(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -562,7 +562,12 @@
         assert b'\x00'.decode('unicode-internal', 'ignore') == ''
 
     def test_backslashreplace(self):
+        import codecs
         assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000'
+        assert b'\x00\x60\x80'.decode(
+            'ascii', 'backslashreplace') == u'\x00\x60\\x80'
+        assert codecs.charmap_decode(
+            b"\x00\x01\x02", "backslashreplace", "ab") == ("ab\\x02", 3)
 
     def test_namereplace(self):
         assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'namereplace') == (
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1432,6 +1432,8 @@
 # This function is also used by _codecs/interp_codecs.py
 (unicode_encode_unicode_escape, raw_unicode_escape_helper
  ) = make_unicode_escape_function()
+(_, raw_unicode_escape_helper_unicode
+) = make_unicode_escape_function(unicode_output=True)
 
 # ____________________________________________________________
 # Raw unicode escape

From pypy.commits at gmail.com  Thu Nov  3 19:52:05 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Thu, 03 Nov 2016 16:52:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Exceptions raised by a codec are
 "wrapped" to indicate which codec is used (like a 'raise X from Y')
Message-ID: <581bcda5.8672c20a.46711.ac8e@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88135:ff527b1b35a4
Date: 2016-11-04 00:49 +0100
http://bitbucket.org/pypy/pypy/changeset/ff527b1b35a4/

Log:	Exceptions raised by a codec are "wrapped" to indicate which codec
	is used (like a 'raise X from Y') But since the new exception has
	the same type as the original, this only works for simple exceptions
	which consist of a single message...

	Also remove code duplication in unicodeobject.py

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -374,7 +374,7 @@
 
 _fmtcache = {}
 _fmtcache2 = {}
-_FMTS = tuple('8NRTds')
+_FMTS = tuple('8NRSTds')
 
 def decompose_valuefmt(valuefmt):
     """Returns a tuple of string parts extracted from valuefmt,
@@ -425,6 +425,8 @@
                         result = str(value).decode('ascii')
                     elif fmt == 'R':
                         result = space.unicode_w(space.repr(value))
+                    elif fmt == 'S':
+                        result = space.unicode_w(space.str(value))
                     elif fmt == 'T':
                         result = space.type(value).name.decode('utf-8')
                     elif fmt == 'N':
@@ -468,6 +470,7 @@
     %8 - The result of arg.decode('utf-8')
     %N - The result of w_arg.getname(space)
     %R - The result of space.unicode_w(space.repr(w_arg))
+    %S - The result of space.unicode_w(space.str(w_arg))
     %T - The result of space.type(w_arg).name
 
     """
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -415,6 +415,34 @@
         state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
 
+# A simplified version of the incredibly complex CPython function
+# _PyErr_TrySetFromCause, which returns a new exception with another
+# error message.  Subclasses of UnicodeErrors are returned inchanged,
+# but this is only a side-effect: they cannot be constructed with a
+# simple message.
+def _wrap_codec_error(space, operr, action, encoding):
+    w_exc = operr.get_w_value(space)
+    try:
+        new_operr = oefmt(space.type(w_exc),
+                          "%s with '%s' codec failed (%T: %S)",
+                          action, encoding, w_exc, w_exc)
+        new_operr.w_cause = w_exc
+        new_operr.normalize_exception(space)
+    except OperationError:
+        # Return the original error
+        return operr
+    return new_operr
+
+def _call_codec(space, w_decoder, w_obj, action, encoding, errors):
+    try:
+        w_res = space.call_function(w_decoder, w_obj, space.wrap(errors))
+    except OperationError as operr:
+        raise _wrap_codec_error(space, operr, action, encoding)
+    if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
+        raise oefmt(space.w_TypeError,
+                    "encoder must return a tuple (object, integer)")
+    return space.getitem(w_res, space.wrap(0))
+
 @unwrap_spec(errors=str)
 def lookup_error(space, errors):
     """lookup_error(errors) -> handler
@@ -448,8 +476,7 @@
     else:
         encoding = space.str_w(w_encoding)
     w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
-    w_res = space.call_function(w_encoder, w_obj, space.wrap(errors))
-    return space.getitem(w_res, space.wrap(0))
+    return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
 
 @unwrap_spec(errors='str_or_None')
 def readbuffer_encode(space, w_data, errors='strict'):
@@ -472,14 +499,7 @@
     else:
         encoding = space.str_w(w_encoding)
     w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1))
-    if space.is_true(w_decoder):
-        w_res = space.call_function(w_decoder, w_obj, space.wrap(errors))
-        if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
-            raise oefmt(space.w_TypeError,
-                        "encoder must return a tuple (object, integer)")
-        return space.getitem(w_res, space.wrap(0))
-    else:
-        assert 0, "XXX, what to do here?"
+    return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
 
 @unwrap_spec(errors=str)
 def register_error(space, errors, w_handler):
@@ -498,6 +518,38 @@
         raise oefmt(space.w_TypeError, "handler must be callable")
 
 # ____________________________________________________________
+# Helpers for unicode.encode() and bytes.decode()
+def lookup_text_codec(space, action, encoding):
+    codec_info = lookup_codec(space, encoding)
+    try:
+        is_text_encoding = space.is_true(
+                space.getattr(codec_info, space.wrap('_is_text_encoding')))
+    except OperationError as e:
+        if e.match(space, space.w_AttributeError):
+            is_text_encoding = True
+        else:
+            raise
+    if not is_text_encoding:
+        raise oefmt(space.w_LookupError,
+                    "'%s' is not a text encoding; "
+                    "use %s to handle arbitrary codecs", encoding, action)
+    return codec_info
+
+def encode_text(space, w_obj, encoding, errors):
+    if errors is None:
+        errors = 'strict'
+    w_encoder = space.getitem(
+        lookup_text_codec(space, "codecs.encode()", encoding), space.wrap(0))
+    return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
+
+def decode_text(space, w_obj, encoding, errors):
+    if errors is None:
+        errors = 'strict'
+    w_decoder = space.getitem(
+        lookup_text_codec(space, "codecs.decode()", encoding), space.wrap(1))
+    return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
+
+# ____________________________________________________________
 # delegation to runicode
 
 from rpython.rlib import runicode
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -376,12 +376,30 @@
         raises(TypeError, b"hello".decode, "test.mytestenc")
         raises(TypeError, "hello".encode, "test.mytestenc")
 
+    def test_codec_wrapped_exception(self):
+        import _codecs
+        def search_function(encoding):
+            def f(input, errors="strict"):
+                raise RuntimeError('should be wrapped')
+            if encoding == 'test.failingenc':
+                return (f, f, None, None)
+            return None
+        _codecs.register(search_function)
+        exc = raises(RuntimeError, b"hello".decode, "test.failingenc")
+        assert str(exc.value) == (
+            "decoding with 'test.failingenc' codec failed "
+            "(RuntimeError: should be wrapped)")
+        exc = raises(RuntimeError, u"hello".encode, "test.failingenc")
+        assert str(exc.value) == (
+            "encoding with 'test.failingenc' codec failed "
+            "(RuntimeError: should be wrapped)")
+
     def test_cpytest_decode(self):
         import codecs
         assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc'
         raises(TypeError, codecs.decode)
         assert codecs.decode(b'abc') == 'abc'
-        raises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii')
+        exc = raises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii')
 
     def test_bad_errorhandler_return(self):
         import codecs
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -577,27 +577,8 @@
         except unicodehelper.RUnicodeEncodeError as ue:
             raise wrap_encode_error(space, ue)
 
-    from pypy.module._codecs.interp_codecs import lookup_codec
-    codec_info = lookup_codec(space, encoding)
-    try:
-        is_text_encoding = space.is_true(
-                space.getattr(codec_info, space.wrap('_is_text_encoding')))
-    except OperationError as e:
-        if e.match(space, space.w_AttributeError):
-            is_text_encoding = True
-        else:
-            raise
-    if not is_text_encoding:
-        raise oefmt(space.w_LookupError,
-                    "'%s' is not a text encoding; "
-                    "use codecs.encode() to handle arbitrary codecs", encoding)
-    w_encoder = space.getitem(codec_info, space.wrap(0))
-    if errors is None:
-        w_errors = space.wrap('strict')
-    else:
-        w_errors = space.wrap(errors)
-    w_restuple = space.call_function(w_encoder, w_object, w_errors)
-    w_retval = space.getitem(w_restuple, space.wrap(0))
+    from pypy.module._codecs.interp_codecs import encode_text
+    w_retval = encode_text(space, w_object, encoding, errors)
     if not space.isinstance_w(w_retval, space.w_bytes):
         raise oefmt(space.w_TypeError,
                     "encoder did not return a bytes object (type '%T')",
@@ -635,27 +616,8 @@
             return space.wrap(str_decode_utf_8(
                     s, len(s), None, final=True, errorhandler=eh)[0])
 
-    from pypy.module._codecs.interp_codecs import lookup_codec
-    codec_info = lookup_codec(space, encoding)
-    try:
-        is_text_encoding = space.is_true(
-                space.getattr(codec_info, space.wrap('_is_text_encoding')))
-    except OperationError as e:
-        if e.match(space, space.w_AttributeError):
-            is_text_encoding = True
-        else:
-            raise
-    if not is_text_encoding:
-        raise oefmt(space.w_LookupError,
-                    "'%s' is not a text encoding; "
-                    "use codecs.decode() to handle arbitrary codecs", encoding)
-    w_decoder = space.getitem(codec_info, space.wrap(1))
-    if errors is None:
-        w_errors = space.wrap('strict')
-    else:
-        w_errors = space.wrap(errors)
-    w_restuple = space.call_function(w_decoder, w_obj, w_errors)
-    w_retval = space.getitem(w_restuple, space.wrap(0))
+    from pypy.module._codecs.interp_codecs import decode_text
+    w_retval = decode_text(space, w_obj, encoding, errors)
     if not space.isinstance_w(w_retval, space.w_unicode):
         raise oefmt(space.w_TypeError,
                     "decoder did not return a bytes object (type '%T')",

From pypy.commits at gmail.com  Fri Nov  4 01:10:55 2016
From: pypy.commits at gmail.com (pjenvey)
Date: Thu, 03 Nov 2016 22:10:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: newinitwarn docs
Message-ID: <581c185f.ca10c20a.1b72a.e22e@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r88136:48c2338313d8
Date: 2016-11-03 22:07 -0700
http://bitbucket.org/pypy/pypy/changeset/48c2338313d8/

Log:	newinitwarn docs

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -96,3 +96,7 @@
 .. branch: ppc-vsx-support
 
 PowerPC implementation for vector operations used in VecOpt
+
+.. branch: newinitwarn
+
+Match CPython's stricter handling of __new/init__ arguments

From pypy.commits at gmail.com  Fri Nov  4 04:35:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 01:35:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: skip tests on 32bit x86 (cpu will never
 try to vectorize there)
Message-ID: <581c483b.04831c0a.7d57e.03d3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88137:4d46a38e4169
Date: 2016-11-04 09:29 +0100
http://bitbucket.org/pypy/pypy/changeset/4d46a38e4169/

Log:	skip tests on 32bit x86 (cpu will never try to vectorize there)

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -1,4 +1,6 @@
 import py
+import sys
+import pytest
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
@@ -17,6 +19,9 @@
 from rpython.jit.tool.oparser import parse as opparse
 from rpython.jit.tool.oparser_model import get_model
 
+if sys.maxint == 2**31-1:
+    pytest.skip("32bit platforms are not supported")
+
 class FakeVecScheduleState(VecScheduleState):
     def __init__(self):
         self.expanded_map = {}
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1,4 +1,5 @@
 import py
+import sys
 import pytest
 
 from rpython.rlib.objectmodel import instantiate
@@ -29,6 +30,9 @@
 
 CPU = getcpuclass()
 
+if sys.maxint == 2**31-1:
+    pytest.skip("32bit platforms are not supported")
+
 class FakeJitDriverStaticData(object):
     vec=True
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -1,4 +1,5 @@
 import py
+import sys
 import pytest
 import math
 import functools
@@ -28,6 +29,9 @@
 
 CPU = getcpuclass()
 
+if sys.maxint == 2**31-1:
+    pytest.skip("32bit platforms are not supported")
+
 @specialize.argtype(0,1)
 def malloc(T,n):
     return lltype.malloc(T, n, flavor='raw', zero=True)
@@ -195,7 +199,7 @@
     test_vec_float_cmp_ne = \
         vec_float_binary(lambda a,b: a != b, rffi.DOUBLE)
 
-    def _vector_simple_int(self, func, type, data):
+    def _vector_simple_int(self, func, type, strat, data):
         func = always_inline(func)
 
         size = rffi.sizeof(type)
@@ -210,11 +214,9 @@
                 raw_storage_setitem(vc, i, rffi.cast(type,c))
                 i += size
 
-        bits = size*8
-        integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1)
-        la = data.draw(st.lists(integers, min_size=10, max_size=150))
+        la = data.draw(st.lists(strat, min_size=10, max_size=150))
         l = len(la)
-        lb = data.draw(st.lists(integers, min_size=l, max_size=l))
+        lb = data.draw(st.lists(strat, min_size=l, max_size=l))
 
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
@@ -229,21 +231,25 @@
         rawstorage.clear()
 
     def vec_int_arith(test_func, arith_func, type):
-        return pytest.mark.parametrize('func,type', [
-            (arith_func, type)
+        size = rffi.sizeof(type)
+        bits = size*8
+        assert 0 <= bits <= 64
+        integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1)
+        return pytest.mark.parametrize('func,type,strat', [
+            (arith_func, type, integers)
         ])(given(data=st.data())(test_func))
 
     vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int)
 
     test_vec_signed_add = \
-        vec_int_arith(lambda a,b: intmask(intmask(a)+intmask(b)), rffi.SIGNED)
+        vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED)
     test_vec_int_add = \
         vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT)
     test_vec_short_add = \
         vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT)
 
     test_vec_signed_sub = \
-        vec_int_arith(lambda a,b: intmask(intmask(a)-intmask(b)), rffi.SIGNED)
+        vec_int_arith(lambda a,b: intmask(a-b), rffi.SIGNED)
     test_vec_sub_int = \
         vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT)
     test_vec_sub_short = \

From pypy.commits at gmail.com  Fri Nov  4 13:13:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Nov 2016 10:13:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix array pickling for typecodes 'q' and
 'Q'
Message-ID: <581cc1b1.94841c0a.69409.d937@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88138:f85b187b80d2
Date: 2016-11-04 17:12 +0000
http://bitbucket.org/pypy/pypy/changeset/f85b187b80d2/

Log:	Fix array pickling for typecodes 'q' and 'Q'

diff --git a/pypy/module/array/reconstructor.py b/pypy/module/array/reconstructor.py
--- a/pypy/module/array/reconstructor.py
+++ b/pypy/module/array/reconstructor.py
@@ -182,6 +182,12 @@
     elif typecode == 'L':
         intsize = rffi.sizeof(rffi.LONG)
         is_signed = False
+    elif typecode == 'q':
+        intsize = rffi.sizeof(rffi.LONGLONG)
+        is_signed = True
+    elif typecode == 'Q':
+        intsize = rffi.sizeof(rffi.LONGLONG)
+        is_signed = False
     else:
         return UNKNOWN_FORMAT
     if intsize == 2:

From pypy.commits at gmail.com  Fri Nov  4 15:35:44 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 12:35:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: add a failing test for s390x
Message-ID: <581ce310.cb091c0a.eeb32.0a0a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88139:92c57b09e8b5
Date: 2016-11-04 18:57 +0100
http://bitbucket.org/pypy/pypy/changeset/92c57b09e8b5/

Log:	add a failing test for s390x

diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -183,7 +183,6 @@
     def _accum_reduce(self, op, arg, accumloc, targetloc):
         # Currently the accumulator can ONLY be 64 bit float/int
         if arg.type == FLOAT:
-            self.mc.VX(targetloc, targetloc, targetloc)
             self.mc.VPDI(targetloc, accumloc, accumloc, permi(1,0))
             if op == '+':
                 self.mc.VFA(targetloc, targetloc, accumloc, l.imm3, l.imm(0b1000), l.imm(0))
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -868,6 +868,36 @@
                   "i{i} = vec_unpack_i(v{v}[8xi8], %d, 1)" % (i % 8)]
             assert self.run_unpack(op, "[16xi8]", {'x': values}, float=False) == v
 
+    def test_int32_float_casts(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            j = 0
+            while i < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(rffi.INT,va,j)
+                b = raw_storage_getitem(rffi.DOUBLE,vb,i)
+                c = rffi.cast(rffi.DOUBLE,a)+b
+                raw_storage_setitem(vc, i, rffi.cast(rffi.DOUBLE,c))
+                j += 4
+                i += 8
+
+        va = alloc_raw_storage(4*30, zero=True)
+        vb = alloc_raw_storage(8*30, zero=True)
+        for i,v in enumerate([1]*30):
+            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+        for i,v in enumerate([-9.0]*30):
+            raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
+        vc = alloc_raw_storage(8*30, zero=True)
+        self.meta_interp(f, [8*30, va, vb, vc], vec=True)
+
+        for i in range(30):
+            assert raw_storage_getitem(rffi.INT,vc,i*8) == 8.0
+
+        free_raw_storage(va)
+        free_raw_storage(vb)
+        free_raw_storage(vc)
+
 
 class TestLLtype(LLJitMixin, VectorizeTests):
     pass

From pypy.commits at gmail.com  Fri Nov  4 15:35:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 12:35:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: resolve an issue. s390x does not put
 the values at the same position in the register as x86 does
Message-ID: <581ce312.031f1c0a.e48ef.10f9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88140:fae41cc92dca
Date: 2016-11-04 20:31 +0100
http://bitbucket.org/pypy/pypy/changeset/fae41cc92dca/

Log:	resolve an issue. s390x does not put the values at the same position
	in the register as x86 does

diff --git a/rpython/jit/backend/llsupport/vector_ext.py b/rpython/jit/backend/llsupport/vector_ext.py
--- a/rpython/jit/backend/llsupport/vector_ext.py
+++ b/rpython/jit/backend/llsupport/vector_ext.py
@@ -208,6 +208,7 @@
 TR_FLOAT_2 = TypeRestrict(FLOAT, 4, 2)
 TR_DOUBLE_2 = TypeRestrict(FLOAT, 8, 2)
 TR_INT32_2 = TypeRestrict(INT, 4, 2)
+TR_INT64_2 = TypeRestrict(INT, 8, 2)
 
 OR_MSTF_I = OpMatchSizeTypeFirst([TR_ANY_INTEGER, TR_ANY_INTEGER])
 OR_MSTF_F = OpMatchSizeTypeFirst([TR_ANY_FLOAT, TR_ANY_FLOAT])
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -2,11 +2,10 @@
 from rpython.jit.metainterp.history import (ConstInt, INT, FLOAT)
 from rpython.jit.backend.llsupport.descr import (ArrayDescr, 
     unpack_arraydescr)
-from rpython.jit.metainterp.resoperation import VectorOp
+from rpython.jit.metainterp.resoperation import VectorOp, rop
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
-from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.backend.zarch.detect_feature import detect_simd_z
 import rpython.jit.backend.zarch.registers as r
 import rpython.jit.backend.zarch.conditions as c
@@ -17,6 +16,8 @@
 from rpython.jit.codewriter import longlong
 from rpython.rlib.objectmodel import always_inline
 from rpython.jit.backend.zarch.arch import WORD
+from rpython.jit.backend.llsupport.vector_ext import (VectorExt,
+        OpRestrict, TR_INT64_2)
 
 def not_implemented(msg):
     msg = '[zarch/vector_ext] %s\n' % msg
@@ -51,6 +52,7 @@
             self.enable(16, accum=True)
             asm.setup_once_vector()
         self._setup = True
+ZSIMDVectorExt.TR_MAPPING[rop.VEC_CAST_INT_TO_FLOAT] = OpRestrict([TR_INT64_2])
 
 class VectorAssembler(object):
     _mixin_ = True
@@ -125,10 +127,18 @@
         self.mc.VX(resloc, loc0, loc1)
 
     def emit_vec_int_signext(self, op, arglocs, regalloc):
-        resloc, loc0 = arglocs
+        resloc, loc0, osizeloc, nsizeloc = arglocs
         # signext is only allowed if the data type sizes do not change.
         # e.g. [byte,byte] = sign_ext([byte, byte]), a simple move is sufficient!
-        self.regalloc_mov(loc0, resloc)
+        osize = osizeloc.value
+        nsize = nsizeloc.value
+        if osize == nsize:
+            self.regalloc_mov(loc0, resloc)
+        elif (osize == 4 and nsize == 8) or (osize == 8 and nsize == 4):
+            self.mc.VLGV(r.SCRATCH, loc0, l.addr(0), l.itemsize_to_mask(osize))
+            self.mc.VLVG(resloc, r.SCRATCH, l.addr(0), l.itemsize_to_mask(nsize))
+            self.mc.VLGV(r.SCRATCH, loc0, l.addr(1), l.itemsize_to_mask(osize))
+            self.mc.VLVG(resloc, r.SCRATCH, l.addr(1), l.itemsize_to_mask(nsize))
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
@@ -291,6 +301,8 @@
         srcidx = srcidxloc.value
         count = countloc.value
         size = sizeloc.value
+        assert isinstance(op, VectorOp)
+        newsize = op.bytesize
         if count == 1:
             if resloc.is_core_reg():
                 assert sourceloc.is_vector_reg()
@@ -301,7 +313,7 @@
                 assert resloc.is_vector_reg()
                 index = l.addr(residx)
                 self.mc.VLR(resloc, vecloc)
-                self.mc.VLVG(resloc, sourceloc, index, l.itemsize_to_mask(size))
+                self.mc.VLVG(resloc, sourceloc, index, l.itemsize_to_mask(newsize))
         else:
             assert resloc.is_vector_reg()
             assert sourceloc.is_vector_reg()
@@ -311,7 +323,7 @@
                 # load from sourceloc into GP reg and store back into resloc
                 self.mc.VLGV(r.SCRATCH, sourceloc, sindex, l.itemsize_to_mask(size))
                 rindex = l.addr(j + residx)
-                self.mc.VLVG(resloc, r.SCRATCH, rindex, l.itemsize_to_mask(size))
+                self.mc.VLVG(resloc, r.SCRATCH, rindex, l.itemsize_to_mask(newsize))
 
     emit_vec_unpack_i = emit_vec_pack_i
 
@@ -461,9 +473,10 @@
     def prepare_vec_int_signext(self, op):
         assert isinstance(op, VectorOp)
         a0 = op.getarg(0)
+        assert isinstance(a0, VectorOp)
         loc0 = self.ensure_vector_reg(a0)
         resloc = self.force_allocate_vector_reg(op)
-        return [resloc, loc0]
+        return [resloc, loc0, imm(a0.bytesize), imm(op.bytesize)]
 
     def prepare_vec_arith_unary(self, op):
         assert isinstance(op, VectorOp)
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -383,8 +383,8 @@
         # 1)
         args[i] = vecop # a)
         assemble_scattered_values(state, pack, args, i) # c)
+        position_values(state, restrict, pack, args, i, pos) # d)
         crop_vector(state, oprestrict, restrict, pack, args, i) # b)
-        position_values(state, restrict, pack, args, i, pos) # d)
         restrict.check(args[i])
 
 def prepare_fail_arguments(state, pack, left, vecop):
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -878,21 +878,21 @@
                 a = raw_storage_getitem(rffi.INT,va,j)
                 b = raw_storage_getitem(rffi.DOUBLE,vb,i)
                 c = rffi.cast(rffi.DOUBLE,a)+b
-                raw_storage_setitem(vc, i, rffi.cast(rffi.DOUBLE,c))
+                raw_storage_setitem(vc, i, c)
                 j += 4
                 i += 8
 
         va = alloc_raw_storage(4*30, zero=True)
         vb = alloc_raw_storage(8*30, zero=True)
         for i,v in enumerate([1]*30):
-            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+            raw_storage_setitem(va, i*4, rffi.cast(rffi.INT,v))
         for i,v in enumerate([-9.0]*30):
             raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
         vc = alloc_raw_storage(8*30, zero=True)
         self.meta_interp(f, [8*30, va, vb, vc], vec=True)
 
         for i in range(30):
-            assert raw_storage_getitem(rffi.INT,vc,i*8) == 8.0
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == -8.0
 
         free_raw_storage(va)
         free_raw_storage(vb)

From pypy.commits at gmail.com  Fri Nov  4 15:35:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 12:35:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-ssl: checkin changes made on the pypy
 stdlib ssl repo
Message-ID: <581ce313.e128c20a.60b07.3035@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88141:d873256da8f4
Date: 2016-11-04 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/d873256da8f4/

Log:	checkin changes made on the pypy stdlib ssl repo

diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
@@ -133,7 +133,7 @@
 int X509_STORE_set_default_paths(X509_STORE *);
 int X509_STORE_set_flags(X509_STORE *, unsigned long);
 void X509_STORE_free(X509_STORE *);
-
+X509_VERIFY_PARAM *_X509_STORE_get0_param(X509_STORE *);
 
 /* X509_STORE_CTX */
 X509_STORE_CTX *X509_STORE_CTX_new(void);
@@ -250,4 +250,8 @@
 static const long Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST = 0;
 static const long X509_V_FLAG_TRUSTED_FIRST = 0;
 #endif
+
+X509_VERIFY_PARAM *_X509_STORE_get0_param(X509_STORE *store) {
+    return store->param;
+}
 """
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -1,3 +1,5 @@
+import sys
+import errno
 import time
 import _thread
 import weakref
@@ -5,7 +7,7 @@
 from _openssl import lib
 from openssl._stdssl.certificate import _test_decode_cert
 from openssl._stdssl.utility import _str_with_len
-from openssl._stdssl.error import (ssl_error,
+from openssl._stdssl.error import (ssl_error, ssl_lib_error,
         SSLError, SSLZeroReturnError, SSLWantReadError,
         SSLWantWriteError, SSLSyscallError,
         SSLEOFError)
@@ -33,6 +35,13 @@
 CLIENT = 0
 SERVER = 1
 
+VERIFY_DEFAULT = 0
+VERIFY_CRL_CHECK_LEAF = lib.X509_V_FLAG_CRL_CHECK 
+VERIFY_CRL_CHECK_CHAIN = lib.X509_V_FLAG_CRL_CHECK | lib.X509_V_FLAG_CRL_CHECK_ALL
+VERIFY_X509_STRICT = lib.X509_V_FLAG_X509_STRICT
+if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
+    VERIFY_X509_TRUSTED_FIRST = lib.X509_V_FLAG_TRUSTED_FIRST
+
 CERT_NONE = 0
 CERT_OPTIONAL = 1
 CERT_REQUIRED = 2
@@ -41,10 +50,13 @@
     if name.startswith('SSL_OP'):
         globals()[name[4:]] = getattr(lib, name)
 
+OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+
 SSL_CLIENT = 0
 SSL_SERVER = 1
 
-PROTOCOL_SSLv2  = 0
+if lib.Cryptography_HAS_SSL2:
+    PROTOCOL_SSLv2  = 0
 PROTOCOL_SSLv3  = 1
 PROTOCOL_SSLv23 = 2
 PROTOCOL_TLSv1    = 3
@@ -71,14 +83,43 @@
 lib.OpenSSL_add_all_algorithms()
 
 class PasswordInfo(object):
-    w_callable = None
+    callable = None
     password = None
     operationerror = None
 PWINFO_STORAGE = {}
 
- at ffi.def_extern
-def _password_callback(buf, size, rwflag, userdata):
-    pass
+def _Cryptography_pem_password_cb(buf, size, rwflag, userdata):
+    pw_info = ffi.from_handle(userdata)
+
+    # TODO PySSL_END_ALLOW_THREADS_S(pw_info->thread_state);
+    password = pw_info.password
+
+    if pw_info.callable:
+        try:
+            password = pw_info.callable()
+        except Exception as e:
+            pw_info.operationerror = e
+            return 0
+
+        if not isinstance(password, (str, bytes, bytearray)):
+            pw_info.operationerror = TypeError("password callback must return a string")
+            return 0
+
+    password = _str_to_ffi_buffer(password)
+
+    if (len(password) > size):
+        pw_info.operationerror = ValueError("password cannot be longer than %d bytes" % size)
+        return 0
+
+    #PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state);
+    ffi.memmove(buf, password, len(password))
+    return len(password)
+
+if lib.Cryptography_STATIC_CALLBACKS:
+    ffi.def_extern(_Cryptography_pem_password_cb)
+    Cryptography_pem_password_cb = lib.Cryptography_pem_password_cb
+else:
+    Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
 
 def _ssl_select(sock, write, timeout):
     pass
@@ -216,7 +257,7 @@
 
 
 class _SSLContext(object):
-    __slots__ = ('ctx', 'check_hostname', 'verify_mode')
+    __slots__ = ('ctx', 'check_hostname')
 
     def __new__(cls, protocol):
         self = object.__new__(cls)
@@ -229,7 +270,7 @@
             method = lib.TLSv1_2_method()
         elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
             method = lib.SSLv3_method()
-        elif protocol == PROTOCOL_SSLv2 and lib.Cryptography_HAS_SSL2_METHOD:
+        elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
             method = lib.SSLv2_method()
         elif protocol == PROTOCOL_SSLv23:
             method = lib.SSLv23_method()
@@ -246,7 +287,7 @@
         # Defaults
         lib.SSL_CTX_set_verify(self.ctx, lib.SSL_VERIFY_NONE, ffi.NULL)
         options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-        if protocol != PROTOCOL_SSLv2:
+        if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
             options |= lib.SSL_OP_NO_SSLv2
         if protocol != PROTOCOL_SSLv3:
             options |= lib.SSL_OP_NO_SSLv3
@@ -269,8 +310,80 @@
                     lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
                 finally:
                     lib.EC_KEY_free(key)
+        if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
+            store = lib.SSL_CTX_get_cert_store(self.ctx)
+            lib.X509_STORE_set_flags(store, lib.X509_V_FLAG_TRUSTED_FIRST)
         return self
 
+    @property
+    def options(self):
+        return lib.SSL_CTX_get_options(self.ctx)
+
+    @options.setter
+    def options(self, value):
+        new_opts = int(value)
+        opts = lib.SSL_CTX_get_options(self.ctx)
+        clear = opts & ~new_opts
+        set = ~opts & new_opts
+        if clear:
+            if lib.Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS:
+                lib.SSL_CTX_clear_options(self.ctx, clear)
+            else:
+                raise ValueError("can't clear options before OpenSSL 0.9.8m")
+        if set:
+            lib.SSL_CTX_set_options(self.ctx, set)
+
+    @property
+    def verify_mode(self):
+        mode = lib.SSL_CTX_get_verify_mode(self.ctx)
+        if mode == lib.SSL_VERIFY_NONE:
+            return CERT_NONE
+        elif mode == lib.SSL_VERIFY_PEER:
+            return CERT_OPTIONAL
+        elif mode == lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
+            return CERT_REQUIRED
+        raise ssl_error("invalid return value from SSL_CTX_get_verify_mode")
+
+    @verify_mode.setter
+    def verify_mode(self, value):
+        n = int(value)
+        if n == CERT_NONE:
+            mode = lib.SSL_VERIFY_NONE
+        elif n == CERT_OPTIONAL:
+            mode = lib.SSL_VERIFY_PEER
+        elif n == CERT_REQUIRED:
+            mode = lib.SSL_VERIFY_PEER | lib.SSL_VERIFY_FAIL_IF_NO_PEER_CERT
+        else:
+            raise ValueError("invalid value for verify_mode")
+        if mode == lib.SSL_VERIFY_NONE and self.check_hostname:
+            raise ValueError("Cannot set verify_mode to CERT_NONE when " \
+                             "check_hostname is enabled.")
+        lib.SSL_CTX_set_verify(self.ctx, mode, ffi.NULL);
+
+    @property
+    def verify_flags(self):
+        store = lib.SSL_CTX_get_cert_store(self.ctx)
+        param = lib._X509_STORE_get0_param(store)
+        flags = lib.X509_VERIFY_PARAM_get_flags(param)
+        return int(flags)
+
+    @verify_flags.setter
+    def verify_flags(self, value):
+        new_flags = int(value)
+        store = lib.SSL_CTX_get_cert_store(self.ctx);
+        param = lib._X509_STORE_get0_param(store)
+        flags = lib.X509_VERIFY_PARAM_get_flags(param);
+        clear = flags & ~new_flags;
+        set = ~flags & new_flags;
+        if clear:
+            param = lib._X509_STORE_get0_param(store)
+            if not lib.X509_VERIFY_PARAM_clear_flags(param, clear):
+                raise ssl_error(None, 0)
+        if set:
+            param = lib._X509_STORE_get0_param(store)
+            if not lib.X509_VERIFY_PARAM_set_flags(param, set):
+                raise ssl_error(None, 0)
+
     def set_ciphers(self, cipherlist):
         cipherlistbuf = _str_to_ffi_buffer(cipherlist)
         ret = lib.SSL_CTX_set_cipher_list(self.ctx, cipherlistbuf)
@@ -294,13 +407,13 @@
             if callable(password):
                 pw_info.callable = password
             else:
-                if isinstance(password, str):
+                if isinstance(password, (str, bytes, bytearray)):
                     pw_info.password = password
+                else:
+                    raise TypeError("password should be a string or callable")
 
-                raise TypeError("password should be a string or callable")
-
-            lib.SSL_CTX_set_default_passwd_cb(self.ctx, _password_callback)
-            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.cast("void*", index))
+            lib.SSL_CTX_set_default_passwd_cb(self.ctx, Cryptography_pem_password_cb)
+            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.new_handle(pw_info))
 
         try:
             certfilebuf = _str_to_ffi_buffer(certfile)
@@ -309,12 +422,12 @@
                 if pw_info.operationerror:
                     lib.ERR_clear_error()
                     raise pw_info.operationerror
-                errno = ffi.errno
-                if errno:
+                _errno = ffi.errno
+                if _errno:
                     lib.ERR_clear_error()
-                    raise OSError(errno, '')
+                    raise OSError(_errno, "Error")
                 else:
-                    raise _ssl_seterror(None, -1)
+                    raise ssl_lib_error()
 
             keyfilebuf = _str_to_ffi_buffer(keyfile)
             ret = lib.SSL_CTX_use_PrivateKey_file(self.ctx, keyfilebuf,
@@ -323,12 +436,12 @@
                 if pw_info.operationerror:
                     lib.ERR_clear_error()
                     raise pw_info.operationerror
-                errno = ffi.errno
-                if errno:
+                _errno = ffi.errno
+                if _errno:
                     lib.ERR_clear_error()
-                    raise OSError(errno, '')
+                    raise OSError(_errno, None)
                 else:
-                    raise _ssl_seterror(None, -1)
+                    raise ssl_lib_error()
 
             ret = lib.SSL_CTX_check_private_key(self.ctx)
             if ret != 1:
@@ -665,12 +778,15 @@
 #
 #
 
+def _str_from_buf(buf):
+    return ffi.string(buf).decode('utf-8')
+
 def _asn1obj2py(obj):
     nid = lib.OBJ_obj2nid(obj)
     if nid == lib.NID_undef:
         raise ValueError("Unknown object")
-    sn = lib.OBJ_nid2sn(nid)
-    ln = lib.OBJ_nid2ln(nid)
+    sn = _str_from_buf(lib.OBJ_nid2sn(nid))
+    ln = _str_from_buf(lib.OBJ_nid2ln(nid))
     buf = ffi.new("char[255]")
     length = lib.OBJ_obj2txt(buf, len(buf), obj, 1)
     if length < 0:
@@ -682,15 +798,23 @@
 
 def txt2obj(txt, name):
     _bytes = _str_to_ffi_buffer(txt)
-    obj = lib.OBJ_txt2obj(_bytes, int(name))
-    if obj is ffi.NULL:
-        raise ValueError("unkown object '%s'", txt)
+    is_name = 0 if name else 1
+    obj = lib.OBJ_txt2obj(_bytes, is_name)
+    if obj == ffi.NULL:
+        raise ValueError("unknown object '%s'" % txt)
     result = _asn1obj2py(obj)
     lib.ASN1_OBJECT_free(obj)
     return result
 
 def nid2obj(nid):
-    raise NotImplementedError
+    if nid < lib.NID_undef:
+        raise ValueError("NID must be positive.")
+    obj = lib.OBJ_nid2obj(nid);
+    if obj == ffi.NULL:
+        raise ValueError("unknown NID %i" % nid)
+    result = _asn1obj2py(obj);
+    lib.ASN1_OBJECT_free(obj);
+    return result;
                                                                
 
 class MemoryBIO(object):
@@ -744,7 +868,7 @@
 #        if (!target) goto error; \
 #    }
     # XXX
-    return ffi.string(buf).decode('utf-8')
+    return ffi.string(buf).decode(sys.getfilesystemencoding())
 
 def get_default_verify_paths():
 
diff --git a/lib_pypy/openssl/_stdssl/error.py b/lib_pypy/openssl/_stdssl/error.py
--- a/lib_pypy/openssl/_stdssl/error.py
+++ b/lib_pypy/openssl/_stdssl/error.py
@@ -23,6 +23,9 @@
 class SSLEOFError(SSLError):
     """ SSL/TLS connection terminated abruptly. """
 
+def ssl_lib_error():
+    errcode = lib.ERR_peek_last_error()
+    return ssl_error(None, 0, None, errcode)
 
 def ssl_error(msg, errno=0, errtype=None, errcode=0):
     reason_str = None
@@ -30,8 +33,8 @@
     if errcode:
         err_lib = lib.ERR_GET_LIB(errcode)
         err_reason = lib.ERR_GET_REASON(errcode)
-        reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
-        lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
+        reason_str = ERR_CODES_TO_NAMES.get((err_lib, err_reason), None)
+        lib_str = LIB_CODES_TO_NAMES.get(err_lib, None)
         msg = ffi.string(lib.ERR_reason_error_string(errcode)).decode('utf-8')
     if not msg:
         msg = "unknown error"
@@ -54,22 +57,16 @@
     #return OperationError(w_exception_class, w_exception)
 
 ERR_CODES_TO_NAMES = {}
+ERR_NAMES_TO_CODES = {}
 LIB_CODES_TO_NAMES = {}
 
-# TODO errcode = error_codes;
-# TODO while (errcode->mnemonic != NULL) {
-# TODO     mnemo = PyUnicode_FromString(errcode->mnemonic);
-# TODO     key = Py_BuildValue("ii", errcode->library, errcode->reason);
-# TODO     if (mnemo == NULL || key == NULL)
-# TODO         return NULL;
-# TODO     if (PyDict_SetItem(err_codes_to_names, key, mnemo))
-# TODO         return NULL;
-# TODO     if (PyDict_SetItem(err_names_to_codes, mnemo, key))
-# TODO         return NULL;
-# TODO     Py_DECREF(key);
-# TODO     Py_DECREF(mnemo);
-# TODO     errcode++;
-# TODO }
+from openssl._stdssl.errorcodes import _error_codes
+
+for mnemo, library, reason in _error_codes:
+    key = (library, reason)
+    assert mnemo is not None and key is not None
+    ERR_CODES_TO_NAMES[key] = mnemo
+    ERR_NAMES_TO_CODES[mnemo] = key
 
 def _fill_and_raise_ssl_error(error, errcode):
     pass

From pypy.commits at gmail.com  Fri Nov  4 16:38:32 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 13:38:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: set right sizes for float to int cast
 in on x86 cpus
Message-ID: <581cf1c8.cb091c0a.746fd.4575@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88143:703d32a5fde7
Date: 2016-11-04 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/703d32a5fde7/

Log:	set right sizes for float to int cast in on x86 cpus

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -159,13 +159,10 @@
         resloc, loc0, loc1, size_loc = arglocs
         size = size_loc.value
         if size == 1:
-            # TODO verify if unsigned subtract is the wanted feature
             self.mc.vsububm(resloc.value, loc0.value, loc1.value)
         elif size == 2:
-            # TODO verify if unsigned subtract is the wanted feature
             self.mc.vsubuhm(resloc.value, loc0.value, loc1.value)
         elif size == 4:
-            # TODO verify if unsigned subtract is the wanted feature
             self.mc.vsubuwm(resloc.value, loc0.value, loc1.value)
         elif size == 8:
             self.mc.vsubudm(resloc.value, loc0.value, loc1.value)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1185,6 +1185,12 @@
     'VEC_INT_SIGNEXT': ('i', 0, 'i', 0, 0),
 }
 
+import platform
+if not platform.machine().startswith('x86'):
+    # Uh, that should be moved to vector_ext really!
+    _cast_ops['CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+    _cast_ops['VEC_CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+
 # ____________________________________________________________
 
 class rop(object):

From pypy.commits at gmail.com  Fri Nov  4 16:38:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 13:38:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: apply the same fix to ppc
Message-ID: <581cf1c7.0370c20a.2d5a9.3c0b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88142:eeebf588aa7e
Date: 2016-11-04 21:30 +0100
http://bitbucket.org/pypy/pypy/changeset/eeebf588aa7e/

Log:	apply the same fix to ppc

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -12,7 +12,6 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.backend.ppc.locations import imm, RegisterLocation
 from rpython.jit.backend.ppc.arch import IS_BIG_ENDIAN
-from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.backend.ppc.arch import PARAM_SAVE_AREA_OFFSET, WORD
 import rpython.jit.backend.ppc.register as r
 import rpython.jit.backend.ppc.condition as c
@@ -22,6 +21,8 @@
 from rpython.jit.codewriter import longlong
 from rpython.jit.backend.ppc.detect_feature import detect_vsx
 from rpython.rlib.objectmodel import always_inline
+from rpython.jit.backend.llsupport.vector_ext import (VectorExt,
+        OpRestrict, TR_INT64_2)
 
 def not_implemented(msg):
     msg = '[ppc/vector_ext] %s\n' % msg
@@ -89,6 +90,7 @@
             self.enable(16, accum=True)
             asm.setup_once_vector()
         self._setup = True
+AltiVectorExt.TR_MAPPING[rop.VEC_CAST_INT_TO_FLOAT] = OpRestrict([TR_INT64_2])
 
 class VectorAssembler(object):
     _mixin_ = True
@@ -201,8 +203,7 @@
             self.mc.xvdivdp(resloc.value, loc0.value, loc1.value)
 
     def emit_vec_int_mul(self, op, arglocs, regalloc):
-        raise NotImplementedError
-        pass # TODO
+        not_implemented("vec_int_mul must never be emitted by the backend")
 
     def emit_vec_int_and(self, op, arglocs, regalloc):
         resloc, loc0, loc1, sizeloc = arglocs
@@ -217,9 +218,42 @@
         self.mc.vxor(resloc.value, loc0.value, loc1.value)
 
     def emit_vec_int_signext(self, op, arglocs, regalloc):
-        resloc, loc0 = arglocs
-        # TODO
-        self.regalloc_mov(loc0, resloc)
+        resloc, loc0, osizeloc, nsizeloc = arglocs
+        # signext is only allowed if the data type sizes do not change.
+        # e.g. [byte,byte] = sign_ext([byte, byte]), a simple move is sufficient!
+        src = loc0.value
+        res = resloc.value
+        osize = osizeloc.value
+        nsize = nsizeloc.value
+        if osize == nsize:
+            self.regalloc_mov(loc0, resloc)
+        else:
+            assert (osize == 4 and nsize == 8) or (osize == 8 and nsize == 4)
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET)
+            self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
+            self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
+            self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
+            for i in range(2): # at most 2 operations
+                if IS_BIG_ENDIAN:
+                    srcoff = (i * osize)
+                    resoff = (i * nsize)
+                else:
+                    if osize == 4:
+                        srcoff = 8 + (i * osize)
+                        resoff = (i * nsize)
+                    else:
+                        srcoff = (i * osize)
+                        resoff = 8 + (i * nsize)
+                if osize == 8:
+                    self.mc.load(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
+                else:
+                    self.mc.lwa(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
+                if nsize == 8:
+                    self.mc.store(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
+                else:
+                    self.mc.stw(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
+
+            self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
@@ -724,9 +758,10 @@
     def prepare_vec_int_signext(self, op):
         assert isinstance(op, VectorOp)
         a0 = op.getarg(0)
+        assert isinstance(a0, VectorOp)
         loc0 = self.ensure_vector_reg(a0)
         resloc = self.force_allocate_vector_reg(op)
-        return [resloc, loc0]
+        return [resloc, loc0, imm(a0.bytesize), imm(op.bytesize)]
 
     def prepare_vec_arith_unary(self, op):
         assert isinstance(op, VectorOp)

From pypy.commits at gmail.com  Fri Nov  4 19:07:43 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 04 Nov 2016 16:07:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix for an annoying issue
Message-ID: <581d14bf.0370c20a.2d5a9.679b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88144:7c88b941e3a8
Date: 2016-11-04 23:07 +0000
http://bitbucket.org/pypy/pypy/changeset/7c88b941e3a8/

Log:	Test and fix for an annoying issue

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -162,6 +162,7 @@
             try:
                 self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
+                operr.record_context(space, frame)
                 return frame.handle_generator_error(operr)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -97,7 +97,6 @@
     def handle_generator_error(self, operr):
         # for generator.py
         ec = self.space.getexecutioncontext()
-        operr.record_context(self.space, self)
         return self.handle_operation_error(ec, operr)
 
     def handle_operation_error(self, ec, operr, attach_tb=True):
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -185,6 +185,27 @@
             tb = tb.tb_next
         assert levels == 3
 
+    def test_throw_context(self):
+        # gen.throw(exc) must not modify exc.__context__
+        def gen():
+            try:
+                yield
+            except:
+                raise ValueError
+
+        try:
+            raise KeyError
+        except KeyError:
+            g = gen()
+            next(g)
+            exc1 = Exception(1)
+            exc2 = Exception(2)
+            exc2.__context__ = exc1
+            try:
+                g.throw(exc2)
+            except ValueError:
+                assert exc2.__context__ is exc1
+
     def test_close(self):
         def f():
             yield 1
@@ -637,7 +658,7 @@
         sys.stderr = io.StringIO()
         gi.close()
         assert 'ZeroDivisionError' in sys.stderr.getvalue()
-    
+
     def test_returning_value_from_delegated_throw(self):
         """
         Test returning value from delegated 'throw'

From pypy.commits at gmail.com  Sat Nov  5 02:34:42 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Fri, 04 Nov 2016 23:34:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: skip vector test on x86 32bit
Message-ID: <581d7d82.958c1c0a.1fb40.20e7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88145:6b8216508ada
Date: 2016-11-05 07:34 +0100
http://bitbucket.org/pypy/pypy/changeset/6b8216508ada/

Log:	skip vector test on x86 32bit

diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -1,5 +1,7 @@
 import py
+import pytest
 import re
+import sys
 from rpython.jit.metainterp import resoperation as rop
 from rpython.jit.metainterp.history import AbstractDescr, AbstractFailDescr
 from rpython.jit.metainterp.history import ConstInt
@@ -96,6 +98,7 @@
       (rop.rop.CAST_SINGLEFLOAT_TO_FLOAT, [VARI], {'from': 4, 'to': 8}),
       (rop.rop.CAST_FLOAT_TO_SINGLEFLOAT, [VARF], {'from': 8, 'to': 4}),
     ])
+ at pytest.mark.skipif("sys.maxint == 2**31-1")
 def test_cast_ops(opnum, args, kwargs):
     op = rop.ResOperation(opnum, args)
     assert op.is_typecast()

From pypy.commits at gmail.com  Sat Nov  5 05:36:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 05 Nov 2016 02:36:21 -0700 (PDT)
Subject: [pypy-commit] cffi default: Don't pollute the namespace with
 'CTypeDescrObject'
Message-ID: <581da815.c1341c0a.c3e5e.5444@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2805:9308471c132a
Date: 2016-11-05 10:36 +0100
http://bitbucket.org/cffi/cffi/changeset/9308471c132a/

Log:	Don't pollute the namespace with 'CTypeDescrObject'

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)

From pypy.commits at gmail.com  Sat Nov  5 10:42:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 05 Nov 2016 07:42:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2428: Fix for an issue with
 ctypes (showing up as
Message-ID: <581defc4.0a22c20a.122a9.6b7d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88146:f59244e00341
Date: 2016-11-05 15:41 +0100
http://bitbucket.org/pypy/pypy/changeset/f59244e00341/

Log:	Issue #2428: Fix for an issue with ctypes (showing up as
	platform.uname() on recent Windows)

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -339,6 +339,9 @@
                         "No symbol %s found in library %s", name, self.name)
         return space.wrap(address_as_uint)
 
+    def getidentifier(self, space):
+        return space.wrap(self.cdll.getidentifier())
+
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
     return space.wrap(W_CDLL(space, name, mode))
@@ -349,6 +352,8 @@
     __new__     = interp2app(descr_new_cdll),
     getfunc     = interp2app(W_CDLL.getfunc),
     getaddressindll = interp2app(W_CDLL.getaddressindll),
+    __int__     = interp2app(W_CDLL.getidentifier),
+    __long__    = interp2app(W_CDLL.getidentifier),
     )
 
 class W_WinDLL(W_CDLL):
diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py
--- a/pypy/module/_rawffi/alt/test/test_funcptr.py
+++ b/pypy/module/_rawffi/alt/test/test_funcptr.py
@@ -643,3 +643,11 @@
         f_name = libfoo.getfunc('AAA_first_ordinal_function', [], types.sint)
         f_ordinal = libfoo.getfunc(1, [], types.sint)
         assert f_name.getaddr() == f_ordinal.getaddr()
+
+    def test_cdll_as_integer(self):
+        import _rawffi
+        from _rawffi.alt import CDLL
+        libfoo = CDLL(self.libfoo_name)
+        A = _rawffi.Array('i')
+        a = A(1, autofree=True)
+        a[0] = libfoo      # should cast libfoo to int/long automatically
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -457,6 +457,9 @@
     def getaddressindll(self, name):
         return dlsym(self.lib, name)
 
+    def getidentifier(self):
+        return rffi.cast(lltype.Unsigned, self.lib)
+
 if os.name == 'nt':
     class WinDLL(CDLL):
         def getpointer(self, name, argtypes, restype, flags=FUNCFLAG_STDCALL):

From pypy.commits at gmail.com  Sat Nov  5 11:05:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 05 Nov 2016 08:05:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2426: reverse the order of
 'a.__dict__'. Gives a more
Message-ID: <581df545.0370c20a.2d5a9.6591@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88147:714537112a18
Date: 2016-11-05 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/714537112a18/

Log:	Issue #2426: reverse the order of 'a.__dict__'. Gives a more
	natural-looking order, and should help performance of copy.copy()

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -830,65 +830,83 @@
     new_obj = map.materialize_r_dict(space, obj, dict_w)
     obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map)
 
+
+class IteratorMixin(object):
+
+    def _init(self, strategy, w_dict):
+        w_obj = strategy.unerase(w_dict.dstorage)
+        self.w_obj = w_obj
+        self.orig_map = curr_map = w_obj._get_mapdict_map()
+        # We enumerate non-lazily the attributes, and store them in the
+        # 'attrs' list.  We then walk that list in opposite order.  This
+        # gives an ordering that is more natural (roughly corresponding
+        # to oldest-first) than the one we get in the direct algorithm
+        # (from leaves to root, looks like backward order).  See issue
+        # #2426: it should improve the performance of code like
+        # copy.copy().
+        attrs = []
+        while True:
+            curr_map = curr_map.search(DICT)
+            if curr_map is None:
+                break
+            attrs.append(curr_map.name)
+            curr_map = curr_map.back
+        self.attrs = attrs
+
+
 class MapDictIteratorKeys(BaseKeyIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseKeyIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_key_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                w_attr = self.space.wrap(attr)
-                return w_attr
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            w_attr = self.space.wrap(attr)
+            return w_attr
         return None
 
 
 class MapDictIteratorValues(BaseValueIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseValueIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_value_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                return self.w_obj.getdictvalue(self.space, attr)
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            return self.w_obj.getdictvalue(self.space, attr)
         return None
 
 
 class MapDictIteratorItems(BaseItemIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseItemIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_item_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None, None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                w_attr = self.space.wrap(attr)
-                return w_attr, self.w_obj.getdictvalue(self.space, attr)
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            w_attr = self.space.wrap(attr)
+            return w_attr, self.w_obj.getdictvalue(self.space, attr)
         return None, None
 
 
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1183,6 +1183,20 @@
         got = a.method()
         assert got == 43
 
+    def test_dict_order(self):
+        # the __dict__ order is not strictly enforced, but in
+        # simple cases like that, we want to follow the order of
+        # creation of the attributes
+        class A(object):
+            pass
+        a = A()
+        a.x = 5
+        a.z = 6
+        a.y = 7
+        assert list(a.__dict__) == ['x', 'z', 'y']
+        assert a.__dict__.values() == [5, 6, 7]
+        assert list(a.__dict__.iteritems()) == [('x', 5), ('z', 6), ('y', 7)]
+
     def test_bug_method_change(self):
         class A(object):
             def method(self):

From pypy.commits at gmail.com  Sat Nov  5 14:30:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 05 Nov 2016 11:30:02 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <581e252a.6252c20a.214f3.c202@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r814:021ba7394e6b
Date: 2016-11-05 19:29 +0100
http://bitbucket.org/pypy/pypy.org/changeset/021ba7394e6b/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66246 of $105000 (63.1%)
+   $66265 of $105000 (63.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $3549 left
+    we have $3567 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Sat Nov  5 16:23:25 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sat, 05 Nov 2016 13:23:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Don't wrap codec exceptions when they
 have an attribute or more arguments.
Message-ID: <581e3fbd.4251c20a.7b642.ebab@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88148:9abb6079ac04
Date: 2016-11-05 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/9abb6079ac04/

Log:	Don't wrap codec exceptions when they have an attribute or more
	arguments.

	Move the code to errors.py because it relies on implementation
	details.

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -344,6 +344,42 @@
         finally:
             self._context_recorded = True
 
+    # A simplified version of _PyErr_TrySetFromCause, which returns a
+    # new exception of the same class, but with another error message.
+    # This only works for exceptions which have just a single message,
+    # and no other attribute.
+    # Otherwise the same OperationError is returned.
+    def try_set_from_cause(self, space, message):
+        from pypy.module.exceptions.interp_exceptions import W_BaseException
+        self.normalize_exception(space)
+        w_value = self.get_w_value(space)
+        if not isinstance(w_value, W_BaseException):
+            return self
+        exc = w_value
+        # "args" should be empty or contain a single string
+        if len(exc.args_w) == 0:
+            pass
+        elif len(exc.args_w) == 1:
+            if not space.isinstance_w(exc.args_w[0], space.w_unicode):
+                return self
+        else:
+            return self
+        # No instance attribute.
+        if exc.w_dict and space.is_true(exc.w_dict):
+            return self
+        # Try to create the new exception.
+        try:
+            new_error = oefmt(space.type(w_value),
+                              "%s (%T: %S)", message, w_value, w_value)
+            new_error.w_cause = w_value
+            new_error.normalize_exception(space)
+            # Copy the traceback, but it does not escape.
+            new_error.set_traceback(self._application_traceback)
+        except OperationError:
+            # Return the original error
+            return self
+        return new_error
+
 
 def _break_context_cycle(space, w_value, w_context):
     """Break reference cycles in the __context__ chain.
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -415,23 +415,11 @@
         state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
 
-# A simplified version of the incredibly complex CPython function
-# _PyErr_TrySetFromCause, which returns a new exception with another
-# error message.  Subclasses of UnicodeErrors are returned inchanged,
-# but this is only a side-effect: they cannot be constructed with a
-# simple message.
 def _wrap_codec_error(space, operr, action, encoding):
-    w_exc = operr.get_w_value(space)
-    try:
-        new_operr = oefmt(space.type(w_exc),
-                          "%s with '%s' codec failed (%T: %S)",
-                          action, encoding, w_exc, w_exc)
-        new_operr.w_cause = w_exc
-        new_operr.normalize_exception(space)
-    except OperationError:
-        # Return the original error
-        return operr
-    return new_operr
+    # Note that UnicodeErrors are not wrapped and returned as is,
+    # "thanks to" a limitation of try_set_from_cause.
+    message = "%s with '%s' codec failed" % (action, encoding)
+    return operr.try_set_from_cause(space, message)
 
 def _call_codec(space, w_decoder, w_obj, action, encoding, errors):
     try:
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -380,11 +380,12 @@
         import _codecs
         def search_function(encoding):
             def f(input, errors="strict"):
-                raise RuntimeError('should be wrapped')
+                raise to_raise
             if encoding == 'test.failingenc':
                 return (f, f, None, None)
             return None
         _codecs.register(search_function)
+        to_raise = RuntimeError('should be wrapped')
         exc = raises(RuntimeError, b"hello".decode, "test.failingenc")
         assert str(exc.value) == (
             "decoding with 'test.failingenc' codec failed "
@@ -393,6 +394,14 @@
         assert str(exc.value) == (
             "encoding with 'test.failingenc' codec failed "
             "(RuntimeError: should be wrapped)")
+        #
+        to_raise.attr = "don't wrap"
+        exc = raises(RuntimeError, u"hello".encode, "test.failingenc")
+        assert exc.value == to_raise
+        #
+        to_raise = RuntimeError("Should", "Not", "Wrap")
+        exc = raises(RuntimeError, u"hello".encode, "test.failingenc")
+        assert exc.value == to_raise
 
     def test_cpytest_decode(self):
         import codecs

From pypy.commits at gmail.com  Sat Nov  5 16:23:27 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sat, 05 Nov 2016 13:23:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Codec error handlers *may* return bytes
 for the replacement string, but most standard ones don't.
Message-ID: <581e3fbf.83071c0a.bbd0d.371d@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88149:0562516d347e
Date: 2016-11-05 21:22 +0100
http://bitbucket.org/pypy/pypy/changeset/0562516d347e/

Log:	Codec error handlers *may* return bytes for the replacement string,
	but most standard ones don't.

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,9 +1,9 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
+from rpython.rlib.rstring import UnicodeBuilder
 from rpython.rlib.runicode import (
     code_to_unichr, MAXUNICODE,
-    raw_unicode_escape_helper, raw_unicode_escape_helper)
+    raw_unicode_escape_helper_unicode)
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -247,7 +247,7 @@
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
         end = space.int_w(w_end)
-        builder = StringBuilder()
+        builder = UnicodeBuilder()
         pos = start
         while pos < end:
             code = ord(obj[pos])
@@ -257,11 +257,11 @@
                 code |= ord(obj[pos+1]) & 0x03FF
                 code += 0x10000
                 pos += 1
-            builder.append("&#")
-            builder.append(str(code))
-            builder.append(";")
+            builder.append(u"&#")
+            builder.append(unicode(str(code)))
+            builder.append(u";")
             pos += 1
-        return space.newtuple([space.newbytes(builder.build()), w_end])
+        return space.newtuple([space.wrap(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
@@ -274,13 +274,13 @@
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
         end = space.int_w(w_end)
-        builder = StringBuilder()
+        builder = UnicodeBuilder()
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            raw_unicode_escape_helper(builder, oc)
+            raw_unicode_escape_helper_unicode(builder, oc)
             pos += 1
-        return space.newtuple([space.newbytes(builder.build()), w_end])
+        return space.newtuple([space.wrap(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
@@ -290,7 +290,7 @@
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            runicode.raw_unicode_escape_helper_unicode(builder, oc)
+            raw_unicode_escape_helper_unicode(builder, oc)
             pos += 1
         return space.newtuple([space.wrap(builder.build()), w_end])
     else:
@@ -304,20 +304,20 @@
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
         end = space.int_w(w_end)
-        builder = StringBuilder()
+        builder = UnicodeBuilder()
         pos = start
         while pos < end:
             oc = ord(obj[pos])
             try:
                 name = unicodedb.name(oc)
             except KeyError:
-                raw_unicode_escape_helper(builder, oc)
+                raw_unicode_escape_helper_unicode(builder, oc)
             else:
-                builder.append('\\N{')
-                builder.append(name)
-                builder.append('}')
+                builder.append(u'\\N{')
+                builder.append(unicode(name))
+                builder.append(u'}')
             pos += 1
-        return space.newtuple([space.newbytes(builder.build()), w_end])
+        return space.newtuple([space.wrap(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)

From pypy.commits at gmail.com  Sun Nov  6 03:14:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 01:14:24 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Issue #2429: be more careful
 in compute_unique_id().
Message-ID: <581ee660.179d1c0a.78789.89fe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r88150:35cfe4e08de5
Date: 2016-11-06 09:12 +0100
http://bitbucket.org/pypy/pypy/changeset/35cfe4e08de5/

Log:	Issue #2429: be more careful in compute_unique_id().

diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -423,6 +423,39 @@
     return obj->h_hash;
 }
 
+RPY_EXTERN
+Signed rpy_reverse_db_cast_ptr_to_int(struct pypy_header0 *obj)
+{
+    /* Returns the "id" of the object.  Should return a unique number
+       among all objects, including prebuilt ones.
+    */
+    if (obj->h_uid == 0) {
+        /* prebuilt object: the object address is good enough, because
+           such addresses should not change between recording and
+           replaying.  The address is even and non-null, so the
+           following formula gives a unique negative result.
+        */
+        return (Signed)(-(((Unsigned)obj) >> 1));
+    }
+    else {
+        /* regular object: for now, may fail to work on 32-bit, where
+           h_uid is a 64-bit number that may grow bigger than 31 bits
+           if the program runs for long enough.  Print a warning if it
+           is the case. */
+        if (((Signed)(obj->h_uid)) != obj->h_uid) {
+            static int warning_printed = 0;
+            if (!warning_printed) {
+                fprintf(stderr, "WARNING: the program executes for long enough "
+                                "that it creates more than 2**31 objects.  In "
+                                "this situation, the id() function may return "
+                                "non-unique results.\n");
+                warning_printed = 1;
+            }
+        }
+        return (Signed)(obj->h_uid);
+    }
+}
+
 static uint64_t recording_offset(void)
 {
     /* must be called with the lock held */
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -231,10 +231,8 @@
 #define OP_REVDB_CALL_DESTRUCTOR(obj, r)                                \
     rpy_reverse_db_call_destructor(obj)
 
-/* Used only for getting a fast hash value that doesn't change too
-   often (with the minimark GC, it changes at most once).  Here,
-   we'll just return the UID. */
-#define RPY_REVDB_CAST_PTR_TO_INT(obj)   (((struct pypy_header0 *)obj)->h_uid)
+#define RPY_REVDB_CAST_PTR_TO_INT(obj)                                  \
+    rpy_reverse_db_cast_ptr_to_int((struct pypy_header0 *)(obj))
 
 #define OP_REVDB_SET_THREAD_BREAKPOINT(tnum, r)                         \
     rpy_reverse_db_set_thread_breakpoint(tnum)
@@ -279,6 +277,7 @@
 RPY_EXTERN void rpy_reverse_db_send_answer(int cmd, int64_t arg1, int64_t arg2,
                                            int64_t arg3, RPyString *extra);
 RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj);
+RPY_EXTERN Signed rpy_reverse_db_cast_ptr_to_int(struct pypy_header0 *obj);
 RPY_EXTERN void rpy_reverse_db_breakpoint(int64_t num);
 RPY_EXTERN long long rpy_reverse_db_get_value(char value_id);
 RPY_EXTERN uint64_t rpy_reverse_db_unique_id_break(void *new_object);

From pypy.commits at gmail.com  Sun Nov  6 12:10:48 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 06 Nov 2016 09:10:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Adapt 'surrogatepass' to utf16 and utf32
 encodings. Other encodings fail.
Message-ID: <581f6418.179d1c0a.78789.3bdd@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88151:0e35b7ce07d1
Date: 2016-11-06 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/0e35b7ce07d1/

Log:	Adapt 'surrogatepass' to utf16 and utf32 encodings. Other encodings
	fail.

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -1,6 +1,7 @@
+import sys
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import UnicodeBuilder
+from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
 from rpython.rlib.runicode import (
     code_to_unichr, MAXUNICODE,
     raw_unicode_escape_helper_unicode)
@@ -322,14 +323,55 @@
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
 
+
+(ENC_UNKNOWN, ENC_UTF8,
+ ENC_UTF16BE, ENC_UTF16LE,
+ ENC_UTF32BE, ENC_UTF32LE) = range(-1, 5)
+BIG_ENDIAN = sys.byteorder == 'big'
+
+STANDARD_ENCODINGS = {
+    'utf8':      (3, ENC_UTF8),
+    'utf_8':     (3, ENC_UTF8),
+    'cp_utf8':   (3, ENC_UTF8),
+    'utf16':     (2, ENC_UTF16BE) if BIG_ENDIAN else (2, ENC_UTF16LE),
+    'utf_16':    (2, ENC_UTF16BE) if BIG_ENDIAN else (2, ENC_UTF16LE),
+    'utf16be':   (2, ENC_UTF16BE),
+    'utf_16be':  (2, ENC_UTF16BE),
+    'utf16_be':  (2, ENC_UTF16BE),
+    'utf_16_be': (2, ENC_UTF16BE),
+    'utf16le':   (2, ENC_UTF16LE),
+    'utf_16le':  (2, ENC_UTF16LE),
+    'utf16_le':  (2, ENC_UTF16LE),
+    'utf_16_le': (2, ENC_UTF16LE),
+    'utf32':     (4, ENC_UTF32BE) if BIG_ENDIAN else (4, ENC_UTF32LE),
+    'utf_32':    (4, ENC_UTF32BE) if BIG_ENDIAN else (4, ENC_UTF32LE),
+    'utf32be':   (4, ENC_UTF32BE),
+    'utf_32be':  (4, ENC_UTF32BE),
+    'utf32_be':  (4, ENC_UTF32BE),
+    'utf_32_be': (4, ENC_UTF32BE),
+    'utf32le':   (4, ENC_UTF32LE),
+    'utf_32le':  (4, ENC_UTF32LE),
+    'utf32_le':  (4, ENC_UTF32LE),
+    'utf_32_le': (4, ENC_UTF32LE),
+}
+
+def get_standard_encoding(encoding):
+    encoding = encoding.lower().replace('-', '_')
+    return STANDARD_ENCODINGS.get(encoding, (0, ENC_UNKNOWN))
+
 def surrogatepass_errors(space, w_exc):
     check_exception(space, w_exc)
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
         obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         w_end = space.getattr(w_exc, space.wrap('end'))
+        encoding = space.str_w(space.getattr(w_exc, space.wrap('encoding')))
+        bytelength, code = get_standard_encoding(encoding)
+        if code == ENC_UNKNOWN:
+            # Not supported, fail with original exception
+            raise OperationError(space.type(w_exc), w_exc)
         end = space.int_w(w_end)
-        res = ''
+        builder = StringBuilder()
         pos = start
         while pos < end:
             ch = ord(obj[pos])
@@ -337,31 +379,61 @@
             if ch < 0xd800 or ch > 0xdfff:
                 # Not a surrogate, fail with original exception
                 raise OperationError(space.type(w_exc), w_exc)
-            res += chr(0xe0 | (ch >> 12))
-            res += chr(0x80 | ((ch >> 6) & 0x3f))
-            res += chr(0x80 | (ch & 0x3f))
-        return space.newtuple([space.newbytes(res), w_end])
+            if code == ENC_UTF8:
+                builder.append(chr(0xe0 | (ch >> 12)))
+                builder.append(chr(0x80 | ((ch >> 6) & 0x3f)))
+                builder.append(chr(0x80 | (ch & 0x3f)))
+            elif code == ENC_UTF16LE:
+                builder.append(chr(ch & 0xff))
+                builder.append(chr(ch >> 8))
+            elif code == ENC_UTF16BE:
+                builder.append(chr(ch >> 8))
+                builder.append(chr(ch & 0xff))
+            elif code == ENC_UTF32LE:
+                builder.append(chr(ch & 0xff))
+                builder.append(chr(ch >> 8))
+                builder.append(chr(0))
+                builder.append(chr(0))
+            elif code == ENC_UTF32BE:
+                builder.append(chr(0))
+                builder.append(chr(0))
+                builder.append(chr(ch >> 8))
+                builder.append(chr(ch & 0xff))
+        return space.newtuple([space.newbytes(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
         start = space.int_w(space.getattr(w_exc, space.wrap('start')))
         obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
+        encoding = space.str_w(space.getattr(w_exc, space.wrap('encoding')))
+        bytelength, code = get_standard_encoding(encoding)
         ch = 0
         # Try decoding a single surrogate character. If there are more,
         # let the codec call us again
-        ch0 = ord(obj[start + 0])
-        ch1 = ord(obj[start + 1])
+        ch0 = ord(obj[start + 0]) if len(obj) > start + 0 else -1
+        ch1 = ord(obj[start + 1]) if len(obj) > start + 1 else -1
         ch2 = ord(obj[start + 2]) if len(obj) > start + 2 else -1
-        if (ch2 != -1 and
-            ch0 & 0xf0 == 0xe0 and
-            ch1 & 0xc0 == 0x80 and
-            ch2 & 0xc0 == 0x80):
-            # it's a three-byte code
-            ch = ((ch0 & 0x0f) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f)
-            if ch < 0xd800 or ch > 0xdfff:
-                # it's not a surrogate - fail
-                ch = 0
+        ch3 = ord(obj[start + 3]) if len(obj) > start + 3 else -1
+        if code == ENC_UTF8:
+            if (ch1 != -1 and ch2 != -1 and
+                ch0 & 0xf0 == 0xe0 and
+                ch1 & 0xc0 == 0x80 and
+                ch2 & 0xc0 == 0x80):
+                # it's a three-byte code
+                ch = ((ch0 & 0x0f) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f)
+        elif code == ENC_UTF16LE:
+            ch = (ch1 << 8) | ch0
+        elif code == ENC_UTF16BE:
+            ch = (ch0 << 8) | ch1
+        elif code == ENC_UTF32LE:
+            ch = (ch3 << 24) | (ch2 << 16) | (ch1 << 8) | ch0
+        elif code == ENC_UTF32BE:
+            ch = (ch0 << 24) | (ch1 << 16) | (ch2 << 8) | ch3
+        if ch < 0xd800 or ch > 0xdfff:
+            # it's not a surrogate - fail
+            ch = 0
         if ch == 0:
             raise OperationError(space.type(w_exc), w_exc)
-        return space.newtuple([space.wrap(unichr(ch)), space.wrap(start + 3)])
+        return space.newtuple([space.wrap(unichr(ch)),
+                               space.wrap(start + bytelength)])
     else:
         raise oefmt(space.w_TypeError,
                     "don't know how to handle %T in error callback", w_exc)
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -624,6 +624,51 @@
         raises(UnicodeDecodeError, b"abc\xed\xa0z".decode, "utf-8",
                "surrogatepass")
 
+    def test_badandgoodsurrogatepassexceptions(self):
+        import codecs
+        surrogatepass_errors = codecs.lookup_error('surrogatepass')
+        # "surrogatepass" complains about a non-exception passed in
+        raises(TypeError, surrogatepass_errors, 42)
+        # "surrogatepass" complains about the wrong exception types
+        raises(TypeError, surrogatepass_errors, UnicodeError("ouch"))
+        # "surrogatepass" can not be used for translating
+        raises(TypeError, surrogatepass_errors,
+               UnicodeTranslateError("\ud800", 0, 1, "ouch"))
+        # Use the correct exception
+        for enc in ("utf-8", "utf-16le", "utf-16be", "utf-32le", "utf-32be"):
+            raises(UnicodeEncodeError, surrogatepass_errors,
+                   UnicodeEncodeError(enc, "a", 0, 1, "ouch"))
+            raises(UnicodeDecodeError, surrogatepass_errors,
+                   UnicodeDecodeError(enc, "a".encode(enc), 0, 1, "ouch"))
+        for s in ("\ud800", "\udfff", "\ud800\udfff"):
+            raises(UnicodeEncodeError, surrogatepass_errors,
+                   UnicodeEncodeError("ascii", s, 0, len(s), "ouch"))
+        tests = [
+            ("utf-8", "\ud800", b'\xed\xa0\x80', 3),
+            ("utf-16le", "\ud800", b'\x00\xd8', 2),
+            ("utf-16be", "\ud800", b'\xd8\x00', 2),
+            ("utf-32le", "\ud800", b'\x00\xd8\x00\x00', 4),
+            ("utf-32be", "\ud800", b'\x00\x00\xd8\x00', 4),
+            ("utf-8", "\udfff", b'\xed\xbf\xbf', 3),
+            ("utf-16le", "\udfff", b'\xff\xdf', 2),
+            ("utf-16be", "\udfff", b'\xdf\xff', 2),
+            ("utf-32le", "\udfff", b'\xff\xdf\x00\x00', 4),
+            ("utf-32be", "\udfff", b'\x00\x00\xdf\xff', 4),
+            ("utf-8", "\ud800\udfff", b'\xed\xa0\x80\xed\xbf\xbf', 3),
+            ("utf-16le", "\ud800\udfff", b'\x00\xd8\xff\xdf', 2),
+            ("utf-16be", "\ud800\udfff", b'\xd8\x00\xdf\xff', 2),
+            ("utf-32le", "\ud800\udfff", b'\x00\xd8\x00\x00\xff\xdf\x00\x00', 4),
+            ("utf-32be", "\ud800\udfff", b'\x00\x00\xd8\x00\x00\x00\xdf\xff', 4),
+        ]
+        for enc, s, b, n in tests:
+            assert surrogatepass_errors(
+                UnicodeEncodeError(enc, "a" + s + "b", 1, 1 + len(s), "ouch")
+            ) == (b, 1 + len(s))
+            assert surrogatepass_errors(
+                UnicodeDecodeError(enc, bytearray(b"a" + b[:n] + b"b"),
+                                   1, 1 + n, "ouch")
+            ) == (s[:1], 1 + n)
+
     def test_badhandler(self):
         import codecs
         results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) )

From pypy.commits at gmail.com  Sun Nov  6 12:10:50 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 06 Nov 2016 09:10:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: epoll() objects now support the `with`
 statement
Message-ID: <581f641a.471ec20a.f430f.3a26@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88152:332f1e3f75da
Date: 2016-11-06 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/332f1e3f75da/

Log:	epoll() objects now support the `with` statement

diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -177,6 +177,13 @@
                 )
             return space.newlist(elist_w)
 
+    def descr_enter(self, space):
+        self.check_closed(space)
+        return self
+
+    def descr_exit(self, space, __args__):
+        self.close()
+
 
 W_Epoll.typedef = TypeDef("select.epoll",
     __new__ = interp2app(W_Epoll.descr__new__.im_func),
@@ -189,5 +196,7 @@
     unregister = interp2app(W_Epoll.descr_unregister),
     modify = interp2app(W_Epoll.descr_modify),
     poll = interp2app(W_Epoll.descr_poll),
+    __enter__ = interp2app(W_Epoll.descr_enter),
+    __exit__ = interp2app(W_Epoll.descr_exit),
 )
 W_Epoll.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -57,6 +57,16 @@
         assert ep.closed
         raises(ValueError, ep.fileno)
 
+    def test_with(self):
+        import select
+
+        ep = select.epoll(16)
+        assert ep.fileno() > 0
+        with ep:
+            assert not ep.closed
+        assert ep.closed
+        raises(ValueError, ep.__enter__)
+
     def test_badcreate(self):
         import select
 

From pypy.commits at gmail.com  Sun Nov  6 12:10:52 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 06 Nov 2016 09:10:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix pickling of SimpleNamespace objects.
Message-ID: <581f641c.8ab81c0a.b2ed.5269@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88153:a221fb64f9a2
Date: 2016-11-06 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/a221fb64f9a2/

Log:	Fix pickling of SimpleNamespace objects.

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -126,6 +126,9 @@
         finally:
             recurse.remove(ident)
 
+# This class is not exposed in sys, but by the types module.
+SimpleNamespace.__module__ = 'types'
+
 
 implementation = SimpleNamespace(
     name='pypy',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -226,6 +226,10 @@
 
 
 class AppTestSysModulePortedFromCPython:
+    spaceconfig = {
+        "usemodules": ["struct"],
+    }
+
     def setup_class(cls):
         cls.w_appdirect = cls.space.wrap(cls.runappdirect)
 
@@ -586,6 +590,13 @@
         raises(AttributeError, "del ns.spam")
         del ns.y
 
+    def test_pickle_simplenamespace(self):
+        import pickle, sys
+        SimpleNamespace = type(sys.implementation)
+        ns = SimpleNamespace(x=1, y=2, w=3)
+        copy = pickle.loads(pickle.dumps(ns))
+        copy.__dict__ == dict(x=1, y=2, w=3, z=4)
+
     def test_reload_doesnt_override_sys_executable(self):
         import sys
         from imp import reload

From pypy.commits at gmail.com  Sun Nov  6 12:10:55 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 06 Nov 2016 09:10:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: The '#c' format is not allowed anymore
Message-ID: <581f641f.857bc20a.e9450.4674@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88155:89f9d1a3ae5a
Date: 2016-11-06 18:09 +0100
http://bitbucket.org/pypy/pypy/changeset/89f9d1a3ae5a/

Log:	The '#c' format is not allowed anymore

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -762,6 +762,10 @@
                 if self._sign != "\0":
                     raise oefmt(space.w_ValueError,
                                 "sign not allowed with 'c' presentation type")
+                if self._alternate:
+                    raise oefmt(space.w_ValueError,
+                                "Alternate form (#) not allowed "
+                                "with 'c' presentation type")
                 value = space.int_w(w_num)
                 max_char = runicode.MAXUNICODE if self.is_unicode else 0xFF
                 if not (0 <= value <= max_char):
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -241,6 +241,7 @@
         assert format(a, "c") == "a"
         raises(ValueError, format, a, "-c")
         raises(ValueError, format, a, ",c")
+        raises(ValueError, format, a, "#c")
         assert format(a, "3c") == "  a"
         assert format(a, "<3c") == "a  "
         assert format(a, "^3c") == " a "

From pypy.commits at gmail.com  Sun Nov  6 12:10:53 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 06 Nov 2016 09:10:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: SimpleNamespace.__eq__
Message-ID: <581f641d.a285c20a.fcc1f.449a@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88154:6c4f77baa08a
Date: 2016-11-06 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/6c4f77baa08a/

Log:	SimpleNamespace.__eq__

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -126,6 +126,10 @@
         finally:
             recurse.remove(ident)
 
+    def __eq__(self, other):
+        return (isinstance(other, SimpleNamespace) and
+                self.__dict__ == other.__dict__)
+
 # This class is not exposed in sys, but by the types module.
 SimpleNamespace.__module__ = 'types'
 
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -595,7 +595,7 @@
         SimpleNamespace = type(sys.implementation)
         ns = SimpleNamespace(x=1, y=2, w=3)
         copy = pickle.loads(pickle.dumps(ns))
-        copy.__dict__ == dict(x=1, y=2, w=3, z=4)
+        assert copy == ns
 
     def test_reload_doesnt_override_sys_executable(self):
         import sys

From pypy.commits at gmail.com  Sun Nov  6 12:38:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 09:38:26 -0800 (PST)
Subject: [pypy-commit] pypy default: Fixes: one file used to let
 RPython-level OSError escape; the other,
Message-ID: <581f6a92.87b2c20a.f4efb.4bc4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88156:33719a46d098
Date: 2016-11-06 18:37 +0100
http://bitbucket.org/pypy/pypy/changeset/33719a46d098/

Log:	Fixes: one file used to let RPython-level OSError escape; the other,
	DLOpenError.

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -325,6 +325,8 @@
             self.cdll = libffi.CDLL(name, mode)
         except DLOpenError as e:
             raise wrap_dlopenerror(space, e, self.name)
+        except OSError as e:
+            raise wrap_oserror(space, e)
 
     def getfunc(self, space, w_name, w_argtypes, w_restype):
         return _getfunc(space, self, w_name, w_argtypes, w_restype)
@@ -376,8 +378,4 @@
 # ========================================================================
 
 def get_libc(space):
-    try:
-        return space.wrap(W_CDLL(space, get_libc_name(), -1))
-    except OSError as e:
-        raise wrap_oserror(space, e)
-
+    return space.wrap(W_CDLL(space, get_libc_name(), -1))
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -224,14 +224,17 @@
             raise oefmt(space.w_ValueError, "Cannot find symbol %s", name)
         return space.wrap(address_as_uint)
 
+def open_cdll(space, name):
+    try:
+        return CDLL(name)
+    except DLOpenError as e:
+        raise wrap_dlopenerror(space, e, name or "<None>")
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
 @unwrap_spec(name='str_or_None')
 def descr_new_cdll(space, w_type, name):
-    try:
-        cdll = CDLL(name)
-    except DLOpenError as e:
-        raise wrap_dlopenerror(space, e, name)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    cdll = open_cdll(space, name)
     return space.wrap(W_CDLL(space, name, cdll))
 
 W_CDLL.typedef = TypeDef(
@@ -620,10 +623,7 @@
 
 def get_libc(space):
     name = get_libc_name()
-    try:
-        cdll = CDLL(name)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    cdll = open_cdll(space, name)
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):

From pypy.commits at gmail.com  Sun Nov  6 13:12:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 10:12:13 -0800 (PST)
Subject: [pypy-commit] pypy default: Pff
Message-ID: <581f727d.857bc20a.e9450.5fc7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88157:2e0882973ed6
Date: 2016-11-06 19:11 +0100
http://bitbucket.org/pypy/pypy/changeset/2e0882973ed6/

Log:	Pff

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -373,6 +373,8 @@
     __new__     = interp2app(descr_new_windll),
     getfunc     = interp2app(W_WinDLL.getfunc),
     getaddressindll = interp2app(W_WinDLL.getaddressindll),
+    __int__     = interp2app(W_CDLL.getidentifier),
+    __long__    = interp2app(W_CDLL.getidentifier),
     )
 
 # ========================================================================

From pypy.commits at gmail.com  Sun Nov  6 13:23:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 10:23:13 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: A branch where any
 uncaught RPython exception is turned into an
Message-ID: <581f7511.231cc20a.37924.5b93@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88158:385d3cf8ef26
Date: 2016-11-06 19:22 +0100
http://bitbucket.org/pypy/pypy/changeset/385d3cf8ef26/

Log:	A branch where any uncaught RPython exception is turned into an app-
	level SystemError


From pypy.commits at gmail.com  Sun Nov  6 13:23:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 10:23:14 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: First draft
Message-ID: <581f7512.c24bc20a.3a44a.5f4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88159:b0bd408d0a1e
Date: 2016-11-06 19:22 +0100
http://bitbucket.org/pypy/pypy/changeset/b0bd408d0a1e/

Log:	First draft

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1836,6 +1836,29 @@
         finally:
             self.sys.track_resources = flag
 
+    def _convert_unexpected_exception_extra(self, e):
+        "NOT_RPYTHON"
+        if e.__class__.__name__ in (
+            'Skipped',     # list of exception class names that are ok
+            ):             # to get during ==untranslated tests== only
+            raise
+        # include the RPython-level traceback
+        exc = sys.exc_info()
+        import traceback, cStringIO
+        f = cStringIO.StringIO()
+        print >> f, "\nTraceback (interpreter-level):"
+        traceback.print_tb(exc[2], file=f)
+        return f.getvalue()
+
+    def _convert_unexpected_exception(self, e):
+        if we_are_translated():
+            extra = ''
+        else:
+            extra = self._convert_unexpected_exception_extra(e)
+        raise OperationError(self.w_SystemError, self.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
+
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -58,10 +58,14 @@
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
         s = self._w_value
-        if self.__class__ is not OperationError and s is None:
-            space = getattr(self.w_type, 'space')
-            if space is not None:
+        space = getattr(self.w_type, 'space', None)
+        if space is not None:
+            if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
+            try:
+                s = space.str_w(s)
+            except Exception:
+                pass
         return '[%s: %s]' % (self.w_type, s)
 
     def errorstr(self, space, use_repr=False):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -712,6 +712,8 @@
             if not we_are_translated():
                 raise
             raise e
+        except OperationError:
+            raise
         except KeyboardInterrupt:
             raise OperationError(space.w_KeyboardInterrupt, space.w_None)
         except MemoryError:
@@ -722,6 +724,8 @@
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
+        except Exception as e:      # general fall-back
+            raise space._convert_unexpected_exception(e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -70,6 +70,8 @@
             next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
         except OperationError as operr:
             next_instr = self.handle_operation_error(ec, operr)
+        except ExitFrame:
+            raise
         except RaiseWithExplicitTraceback as e:
             next_instr = self.handle_operation_error(ec, e.operr,
                                                      attach_tb=False)
@@ -85,6 +87,8 @@
             next_instr = self.handle_asynchronous_error(ec,
                 self.space.w_RuntimeError,
                 self.space.wrap("maximum recursion depth exceeded"))
+        except Exception as e:      # general fall-back
+            raise self.space._convert_unexpected_exception(e)
         return next_instr
 
     def handle_asynchronous_error(self, ec, w_type, w_value=None):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -1,8 +1,10 @@
 import py
 from pypy.interpreter import executioncontext
+from pypy.interpreter.error import OperationError
 
-class Finished(Exception):
-    pass
+class Finished(OperationError):
+    def __init__(self):
+        OperationError.__init__(self, "exception_class", "exception_value")
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -4,6 +4,7 @@
 from pypy.interpreter import gateway, argument
 from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.error import OperationError
 import py
 import sys
 
@@ -771,6 +772,21 @@
         w_g = space.wrap(gateway.interp2app_temp(g, doc='bar'))
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
+    def test_system_error(self):
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        e = py.test.raises(OperationError, space.appexec, [w_g], """(my_g):
+            my_g()
+        """)
+        err = str(e.value)
+        assert 'SystemError' in err
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist

From pypy.commits at gmail.com  Sun Nov  6 13:35:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 10:35:45 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix test_nice to not expect a niceness
 level of 22 if we start with 19, 
Message-ID: <581f7801.838e1c0a.b74c.ab92@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88160:7305f377312a
Date: 2016-11-06 19:35 +0100
http://bitbucket.org/pypy/pypy/changeset/7305f377312a/

Log:	Fix test_nice to not expect a niceness level of 22 if we start with
	19, which is the maximum generally on Posix

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1015,7 +1015,8 @@
             pid1, status1 = os.waitpid(pid, 0)
             assert pid1 == pid
             assert os.WIFEXITED(status1)
-            assert os.WEXITSTATUS(status1) == myprio + 3
+            expected = min(myprio + 3, 19)
+            assert os.WEXITSTATUS(status1) == expected
 
     if hasattr(os, 'symlink'):
         def test_symlink(self):

From pypy.commits at gmail.com  Sun Nov  6 13:42:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 10:42:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Add a test for 2e0882973ed6
Message-ID: <581f7979.ca57c20a.b99fd.2443@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88161:8430d9ee0092
Date: 2016-11-06 19:41 +0100
http://bitbucket.org/pypy/pypy/changeset/8430d9ee0092/

Log:	Add a test for 2e0882973ed6

diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py
--- a/pypy/module/_rawffi/alt/test/test_funcptr.py
+++ b/pypy/module/_rawffi/alt/test/test_funcptr.py
@@ -651,3 +651,13 @@
         A = _rawffi.Array('i')
         a = A(1, autofree=True)
         a[0] = libfoo      # should cast libfoo to int/long automatically
+
+    def test_windll_as_integer(self):
+        if not self.iswin32:
+            skip("windows specific")
+        import _rawffi
+        from _rawffi.alt import WinDLL
+        libm = WinDLL(self.libm_name)
+        A = _rawffi.Array('i')
+        a = A(1, autofree=True)
+        a[0] = libm        # should cast libm to int/long automatically

From pypy.commits at gmail.com  Sun Nov  6 13:49:35 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 Nov 2016 10:49:35 -0800 (PST)
Subject: [pypy-commit] pypy default: hack to let setuptools find the msvc
 compiler (maybe), document Windows 10 experience
Message-ID: <581f7b3f.43e61c0a.13c93.eb1a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88162:12160ae9198a
Date: 2016-11-06 20:48 +0200
http://bitbucket.org/pypy/pypy/changeset/12160ae9198a/

Log:	hack to let setuptools find the msvc compiler (maybe), document
	Windows 10 experience

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -20,6 +20,22 @@
 
 .. _our downloads: http://pypy.org/download.html
 
+Installing Visual Compiler v9 (for Python 2.7)
+----------------------------------------------
+
+This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
+was checked in Nov 2016). Note that the compiler suite will be installed in
+``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
+Using a current version of ``setuptools`` will be able to find it there. For
+Windows 10, you must right-click the download, and under ``Properties`` ->
+``Compatibility`` mark it as ``Run run this program in comatibility mode for``
+``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
+otherwise ``mt.exe`` will silently fail. Installation will begin automatically
+by running the mt.exe command by hand from a DOS window (that is how the author
+discovered the problem).
+
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
 
 Translating PyPy with Visual Studio
 -----------------------------------
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -35,18 +35,27 @@
     return _get_compiler_type(cc, True)
 
 def _get_msvc_env(vsver, x64flag):
+    vcvars = None
     try:
         toolsdir = os.environ['VS%sCOMNTOOLS' % vsver]
     except KeyError:
-        return None
+        # try to import from the "Microsoft Visual C++ Compiler for Python 2.7"
+        try:
+            import setuptools
+        except ImportError:
+            return None
+        # XXX works for 90 but is it generalizable?
+        toolsdir = ''
+        vcvars = setuptools.msvc.msvc9_find_vcvarsall(vsver/10)
 
-    if x64flag:
-        vsinstalldir = os.path.abspath(os.path.join(toolsdir, '..', '..'))
-        vcinstalldir = os.path.join(vsinstalldir, 'VC')
-        vcbindir = os.path.join(vcinstalldir, 'BIN')
-        vcvars = os.path.join(vcbindir, 'amd64', 'vcvarsamd64.bat')
-    else:
-        vcvars = os.path.join(toolsdir, 'vsvars32.bat')
+    if not vcvars:
+        if x64flag:
+            vsinstalldir = os.path.abspath(os.path.join(toolsdir, '..', '..'))
+            vcinstalldir = os.path.join(vsinstalldir, 'VC')
+            vcbindir = os.path.join(vcinstalldir, 'BIN')
+            vcvars = os.path.join(vcbindir, 'amd64', 'vcvarsamd64.bat')
+        else:
+            vcvars = os.path.join(toolsdir, 'vsvars32.bat')
 
     import subprocess
     try:

From pypy.commits at gmail.com  Sun Nov  6 14:07:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 11:07:32 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: Dump the
 RPython-level traceback up to the point where we caught it
Message-ID: <581f7f74.0209c20a.36aab.72cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88163:9ceb823c232b
Date: 2016-11-06 20:06 +0100
http://bitbucket.org/pypy/pypy/changeset/9ceb823c232b/

Log:	Dump the RPython-level traceback up to the point where we caught it

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,7 +3,7 @@
 from rpython.rlib.cache import Cache
 from rpython.tool.uid import HUGEVAL_BYTES
 from rpython.rlib import jit, types
-from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.debug import make_sure_not_resized, debug_print_traceback
 from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
      compute_unique_id, specialize)
 from rpython.rlib.signature import signature
@@ -1852,7 +1852,8 @@
 
     def _convert_unexpected_exception(self, e):
         if we_are_translated():
-            extra = ''
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
         else:
             extra = self._convert_unexpected_exception_extra(e)
         raise OperationError(self.w_SystemError, self.wrap(
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -11,7 +11,7 @@
 
 # Expose these here (public interface)
 from rpython.rtyper.debug import (
-    ll_assert, FatalError, fatalerror, fatalerror_notb)
+    ll_assert, FatalError, fatalerror, fatalerror_notb, debug_print_traceback)
 
 
 class DebugLog(list):
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -118,3 +118,30 @@
     finally:
         debug._log = None
     assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+
+
+def test_debug_print_traceback():
+    from rpython.translator.c.test.test_genc import compile
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+
+    def ggg(n):
+        if n < 10:
+            ggg(n + 1)
+        else:
+            raise ValueError
+    def recovery():
+        llop.debug_print_traceback(lltype.Void)
+    recovery._dont_inline_ = True
+    def fff():
+        try:
+            ggg(0)
+        except:
+            recovery()
+
+    fn = compile(fff, [], return_stderr=True)
+    stderr = fn()
+    assert 'RPython traceback:\n' in stderr
+    assert stderr.count('entry_point') == 1
+    assert stderr.count('ggg') == 11
+    assert stderr.count('recovery') == 0
diff --git a/rpython/rtyper/debug.py b/rpython/rtyper/debug.py
--- a/rpython/rtyper/debug.py
+++ b/rpython/rtyper/debug.py
@@ -45,3 +45,12 @@
 fatalerror_notb._dont_inline_ = True
 fatalerror_notb._jit_look_inside_ = False
 fatalerror_notb._annenforceargs_ = [str]
+
+def debug_print_traceback():
+    # print to stderr the RPython traceback of the last caught exception,
+    # but without interrupting the program
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.debug_print_traceback(lltype.Void)
+debug_print_traceback._dont_inline_ = True
+debug_print_traceback._jit_look_inside_ = False
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -53,7 +53,8 @@
                                            unsigned_ffffffff)
 
 def compile(fn, argtypes, view=False, gcpolicy="none", backendopt=True,
-            annotatorpolicy=None, thread=False, **kwds):
+            annotatorpolicy=None, thread=False,
+            return_stderr=False, **kwds):
     argtypes_unroll = unrolling_iterable(enumerate(argtypes))
 
     for argtype in argtypes:
@@ -139,7 +140,8 @@
 
         stdout = t.driver.cbuilder.cmdexec(
             " ".join([llrepr_in(arg) for arg in args]),
-            expect_crash=(expected_exception_name is not None))
+            expect_crash=(expected_exception_name is not None),
+            err=return_stderr)
         #
         if expected_exception_name is not None:
             stdout, stderr = stdout
@@ -154,6 +156,8 @@
             assert lastline == expected or prevline == expected
             return None
 
+        if return_stderr:
+            stdout, stderr = stdout
         output(stdout)
         stdout, lastline, empty = stdout.rsplit('\n', 2)
         assert empty == ''
@@ -168,6 +172,8 @@
         else:
             assert mallocs - frees in expected_extra_mallocs
         #
+        if return_stderr:
+            return stderr
         if ll_res in [lltype.Signed, lltype.Unsigned, lltype.SignedLongLong,
                       lltype.UnsignedLongLong]:
             return int(res)

From pypy.commits at gmail.com  Sun Nov  6 14:11:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 Nov 2016 11:11:41 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: Add a way to get
 an RPython-level exception
Message-ID: <581f806d.876ec20a.12fc1.5af3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88164:37f15c8c0ecf
Date: 2016-11-06 20:11 +0100
http://bitbucket.org/pypy/pypy/changeset/37f15c8c0ecf/

Log:	Add a way to get an RPython-level exception

diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -86,7 +86,9 @@
         return space.w_None
     return space.get(w_descr, w_obj)
 
-def do_what_I_mean(space):
+def do_what_I_mean(space, w_crash=None):
+    if not space.is_none(w_crash):
+        raise ValueError    # RPython-level, uncaught
     return space.wrap(42)
 
 
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -92,6 +92,7 @@
         from __pypy__ import do_what_I_mean
         x = do_what_I_mean()
         assert x == 42
+        raises(SystemError, do_what_I_mean, 1)
 
     def test_list_strategy(self):
         from __pypy__ import strategy

From pypy.commits at gmail.com  Mon Nov  7 04:31:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 07 Nov 2016 01:31:24 -0800 (PST)
Subject: [pypy-commit] pypy default: test fixes
Message-ID: <582049ec.c6bdc20a.bc113.5af6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88165:c43d1c913296
Date: 2016-11-07 10:29 +0100
http://bitbucket.org/pypy/pypy/changeset/c43d1c913296/

Log:	test fixes

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_costmodel.py
@@ -172,16 +172,20 @@
         assert savings == 6
 
     def test_load_arith_store(self):
+        import platform
+        size = 4
+        if not platform.machine().startswith('x86'):
+            size = 8
         loop1 = self.parse_trace("""
         f10 = raw_load_f(p0, i0, descr=double)
         f11 = raw_load_f(p0, i1, descr=double)
         i20 = cast_float_to_int(f10)
         i21 = cast_float_to_int(f11)
-        i30 = int_signext(i20, 4)
-        i31 = int_signext(i21, 4)
+        i30 = int_signext(i20, {size})
+        i31 = int_signext(i21, {size})
         raw_store(p0, i3, i30, descr=int)
         raw_store(p0, i4, i31, descr=int)
-        """)
+        """.format(size=size))
         savings = self.savings(loop1)
         assert savings >= 0
 
@@ -195,7 +199,7 @@
         savings = self.savings(loop1)
         assert savings == 2
 
-    @py.test.mark.parametrize("bytes,s", [(1,0),(2,0),(4,0),(8,0)])
+    @py.test.mark.parametrize("bytes,s", [(4,0),(8,0)])
     def test_sum_float_to_int(self, bytes, s):
         loop1 = self.parse_trace("""
         f10 = raw_load_f(p0, i0, descr=double)
@@ -254,7 +258,7 @@
         except NotAProfitableLoop:
             pass
 
-    def test_force_long_to_int_cast(self):
+    def test_force_int_to_float_cast(self):
         trace = self.parse_trace("""
         i10 = raw_load_i(p0, i1, descr=long)
         i11 = raw_load_i(p0, i2, descr=long)
@@ -262,7 +266,7 @@
         f11 = cast_int_to_float(i11)
         """)
         number = self.savings(trace)
-        assert number == 1
+        assert number >= 1
 
 
 class Test(CostModelBaseTest, LLtypeMixin):
diff --git a/rpython/jit/metainterp/test/test_resoperation.py b/rpython/jit/metainterp/test/test_resoperation.py
--- a/rpython/jit/metainterp/test/test_resoperation.py
+++ b/rpython/jit/metainterp/test/test_resoperation.py
@@ -1,5 +1,6 @@
 import py
 import pytest
+import platform
 import re
 import sys
 from rpython.jit.metainterp import resoperation as rop
@@ -92,12 +93,15 @@
 
 VARI = rop.InputArgInt()
 VARF = rop.InputArgFloat()
- at py.test.mark.parametrize('opnum,args,kwargs', 
-    [ (rop.rop.INT_SIGNEXT, [VARI, ConstInt(2)], {'from': INT_WORD, 'to': 2, 'cast_to': ('i', 2) }),
-      (rop.rop.CAST_FLOAT_TO_INT, [VARF], {'from': 8, 'to': 4}),
-      (rop.rop.CAST_SINGLEFLOAT_TO_FLOAT, [VARI], {'from': 4, 'to': 8}),
-      (rop.rop.CAST_FLOAT_TO_SINGLEFLOAT, [VARF], {'from': 8, 'to': 4}),
-    ])
+args = [ (rop.rop.INT_SIGNEXT, [VARI, ConstInt(2)], {'from': INT_WORD, 'to': 2, 'cast_to': ('i', 2) }),
+         (rop.rop.CAST_FLOAT_TO_INT, [VARF], {'from': 8, 'to': 4}),
+         (rop.rop.CAST_SINGLEFLOAT_TO_FLOAT, [VARI], {'from': 4, 'to': 8}),
+         (rop.rop.CAST_FLOAT_TO_SINGLEFLOAT, [VARF], {'from': 8, 'to': 4}),
+        ]
+if not platform.machine().startswith('x86'):
+    del args[1]
+    args.append((rop.rop.CAST_FLOAT_TO_INT, [VARF], {'from': 8, 'to': 8}))
+ at py.test.mark.parametrize('opnum,args,kwargs', args)
 @pytest.mark.skipif("sys.maxint == 2**31-1")
 def test_cast_ops(opnum, args, kwargs):
     op = rop.ResOperation(opnum, args)
@@ -106,6 +110,7 @@
     assert op.cast_to_bytesize() == kwargs['to']
     if 'cast_to' in kwargs:
         assert op.cast_to() == kwargs['cast_to']
+del args
 
 def test_unpack_1():
     op = rop.ResOperation(rop.rop.VEC_UNPACK_I,

From pypy.commits at gmail.com  Mon Nov  7 08:14:26 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 05:14:26 -0800 (PST)
Subject: [pypy-commit] pypy default: merge branch to fix test_whatsnew
Message-ID: <58207e32.8b901c0a.4e4a5.148b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88166:86f043dfd863
Date: 2016-11-07 13:59 +0200
http://bitbucket.org/pypy/pypy/changeset/86f043dfd863/

Log:	merge branch to fix test_whatsnew


From pypy.commits at gmail.com  Mon Nov  7 08:14:27 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 05:14:27 -0800 (PST)
Subject: [pypy-commit] pypy default: fix whatsnew,
 fix test for windows (with modified command.py to allow test to run)
Message-ID: <58207e33.e626c20a.4cc77.ba73@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88167:f826843099a7
Date: 2016-11-07 14:07 +0200
http://bitbucket.org/pypy/pypy/changeset/f826843099a7/

Log:	fix whatsnew, fix test for windows (with modified command.py to
	allow test to run)

diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -110,4 +110,4 @@
         "hg log -r %s --template '{branch}'" % startrev)
     if errcode != 0:
         py.test.skip('no Mercurial repo')
-    assert wc_branch == 'default'
+    assert wc_branch in ('default', "'default'") # sometimes the ' leaks (windows)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -100,3 +100,8 @@
 .. branch: newinitwarn
 
 Match CPython's stricter handling of __new/init__ arguments
+
+.. branch: openssl-1.1
+
+Support for OpenSSL version 1.1 (in addition to version 1.0).
+Tested on Linux (1.1, 1.0), on Win32, and Mac (1.0 only)

From pypy.commits at gmail.com  Mon Nov  7 08:14:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 05:14:29 -0800 (PST)
Subject: [pypy-commit] pypy default: restart default version
Message-ID: <58207e35.6252c20a.214f3.c420@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88168:7e9787939641
Date: 2016-11-07 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/7e9787939641/

Log:	restart default version

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-pypy2-5.6.0.rst
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.12"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.5.0-alpha0"
-#define PYPY_VERSION_NUM  0x05050000
+#define PYPY_VERSION "5.7.0-alpha0"
+#define PYPY_VERSION_NUM  0x05070000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 5, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Mon Nov  7 08:14:31 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 05:14:31 -0800 (PST)
Subject: [pypy-commit] pypy default: restart whatsnew, update contributors
Message-ID: <58207e37.ca13c20a.6c39c.bb96@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88169:dd903dfe3a09
Date: 2016-11-07 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/dd903dfe3a09/

Log:	restart whatsnew, update contributors

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -9,8 +9,8 @@
   Philip Jenvey
   Ronan Lamy
   Brian Kearns
+  Richard Plangger
   Michael Hudson
-  Richard Plangger
   Manuel Jacob
   David Schneider
   Holger Krekel
@@ -54,10 +54,10 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
-  Spenser Bauman
   Tyler Wade
   Vincent Legoll
   Michael Foord
@@ -65,13 +65,13 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Stefano Rivera
   Timo Paulssen
   Edd Barrett
   Squeaky
@@ -89,12 +89,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -128,10 +128,11 @@
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -147,17 +148,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
   Daniel Patrick
-  Jasper Schulz
-  Christian Hudon
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -165,8 +165,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -175,11 +175,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -196,7 +197,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -219,6 +219,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -231,6 +232,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -258,9 +260,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -274,10 +274,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
-  "Aaron Gallagher
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -287,8 +287,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -302,13 +303,11 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
-  JohnDoe
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
   reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
@@ -316,25 +315,24 @@
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
   remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
   werat
-  hgattic
+
+
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -32,7 +32,7 @@
     'Niklaus Haldimann': ['nik'],
     'Alexander Schremmer': ['xoraxax'],
     'Anders Hammarquist': ['iko'],
-    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'David Edelsohn': ['edelsoh', 'edelsohn','opassembler.py'],
     'Niko Matsakis': ['niko'],
     'Jakub Gustak': ['jlg'],
     'Guido Wesdorp': ['guido'],
@@ -75,6 +75,9 @@
     'Spenser Bauman':['Spenser Andrew Bauman'],
     'Raffael Tfirst':['raffael.tfirst at gmail.com'],
     'timo':['timo at eistee.fritz.box'],
+    'Jasper Schulz':['Jasper.Schulz'],
+    'Aaron Gallagher':['"Aaron Gallagher'],
+    'Yasir Suhail':['yasirs'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,8 @@
+==========================
+What's new in PyPy2.7 5.6+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
+
+

From pypy.commits at gmail.com  Mon Nov  7 08:14:32 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 05:14:32 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: branch,
 update ver for release
Message-ID: <58207e38.a285c20a.fcc1f.c539@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88170:64fff7b2dc26
Date: 2016-11-07 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/64fff7b2dc26/

Log:	branch, update ver for release

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.12"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.7.0-alpha0"
-#define PYPY_VERSION_NUM  0x05070000
+#define PYPY_VERSION "5.6.0"
+#define PYPY_VERSION_NUM  0x05060000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 6, 0, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Mon Nov  7 08:52:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 05:52:37 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: Move the logic to
 pyframe. Should avoid the ContinueRunningNormally
Message-ID: <58208725.41d71c0a.85aa6.2185@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88171:04a9363395d5
Date: 2016-11-07 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/04a9363395d5/

Log:	Move the logic to pyframe. Should avoid the ContinueRunningNormally
	exception seen with the JIT

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,7 +3,7 @@
 from rpython.rlib.cache import Cache
 from rpython.tool.uid import HUGEVAL_BYTES
 from rpython.rlib import jit, types
-from rpython.rlib.debug import make_sure_not_resized, debug_print_traceback
+from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.objectmodel import (we_are_translated, newlist_hint,
      compute_unique_id, specialize)
 from rpython.rlib.signature import signature
@@ -1836,30 +1836,6 @@
         finally:
             self.sys.track_resources = flag
 
-    def _convert_unexpected_exception_extra(self, e):
-        "NOT_RPYTHON"
-        if e.__class__.__name__ in (
-            'Skipped',     # list of exception class names that are ok
-            ):             # to get during ==untranslated tests== only
-            raise
-        # include the RPython-level traceback
-        exc = sys.exc_info()
-        import traceback, cStringIO
-        f = cStringIO.StringIO()
-        print >> f, "\nTraceback (interpreter-level):"
-        traceback.print_tb(exc[2], file=f)
-        return f.getvalue()
-
-    def _convert_unexpected_exception(self, e):
-        if we_are_translated():
-            debug_print_traceback()
-            extra = '; internal traceback was dumped to stderr'
-        else:
-            extra = self._convert_unexpected_exception_extra(e)
-        raise OperationError(self.w_SystemError, self.wrap(
-            "unexpected internal exception (please report a bug): %r%s" %
-            (e, extra)))
-
 
 class AppExecCache(SpaceCache):
     def build(cache, source):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -712,8 +712,6 @@
             if not we_are_translated():
                 raise
             raise e
-        except OperationError:
-            raise
         except KeyboardInterrupt:
             raise OperationError(space.w_KeyboardInterrupt, space.w_None)
         except MemoryError:
@@ -724,8 +722,6 @@
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
-        except Exception as e:      # general fall-back
-            raise space._convert_unexpected_exception(e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,6 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+import sys
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
@@ -276,8 +277,13 @@
                     next_instr = r_uint(self.last_instr + 1)
                     if next_instr != 0:
                         self.pushvalue(w_inputvalue)
-                w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                            executioncontext)
+                try:
+                    w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                                executioncontext)
+                except OperationError:
+                    raise
+                except Exception as e:      # general fall-back
+                    raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
@@ -883,6 +889,34 @@
             frame = frame.f_backref()
         return None
 
+    def _convert_unexpected_exception_extra(self, e):
+        "NOT_RPYTHON"
+        if e.__class__.__name__ in (
+            'Skipped',     # list of exception class names that are ok
+            ):             # to get during ==untranslated tests== only
+            raise
+        # include the RPython-level traceback
+        exc = sys.exc_info()
+        import traceback, cStringIO
+        f = cStringIO.StringIO()
+        print >> f, "\nTraceback (interpreter-level):"
+        traceback.print_tb(exc[2], file=f)
+        return f.getvalue()
+
+    def _convert_unexpected_exception(self, e):
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
+        else:
+            extra = self._convert_unexpected_exception_extra(e)
+        operr = OperationError(self.space.w_SystemError, self.space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
+        pytraceback.record_application_traceback(
+            self.space, operr, self, self.last_instr)
+        raise operr
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -70,8 +70,6 @@
             next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
         except OperationError as operr:
             next_instr = self.handle_operation_error(ec, operr)
-        except ExitFrame:
-            raise
         except RaiseWithExplicitTraceback as e:
             next_instr = self.handle_operation_error(ec, e.operr,
                                                      attach_tb=False)
@@ -87,8 +85,6 @@
             next_instr = self.handle_asynchronous_error(ec,
                 self.space.w_RuntimeError,
                 self.space.wrap("maximum recursion depth exceeded"))
-        except Exception as e:      # general fall-back
-            raise self.space._convert_unexpected_exception(e)
         return next_instr
 
     def handle_asynchronous_error(self, ec, w_type, w_value=None):

From pypy.commits at gmail.com  Mon Nov  7 08:53:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 07 Nov 2016 05:53:47 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _io
Message-ID: <5820876b.4b8b1c0a.588b4.208c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88172:d4be9a5ff016
Date: 2016-11-07 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/d4be9a5ff016/

Log:	_io

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -28,9 +28,9 @@
     w_type = space.gettypeobject(W_BlockingIOError.typedef)
     w_value = space.call_function(
         w_type,
-        space.wrap(rposix.get_saved_errno()),
-        space.wrap("write could not complete without blocking"),
-        space.wrap(written))
+        space.newint(rposix.get_saved_errno()),
+        space.newtext("write could not complete without blocking"),
+        space.newint(written))
     return OperationError(w_type, w_value)
 
 
@@ -62,14 +62,14 @@
 class W_BufferedIOBase(W_IOBase):
     def _unsupportedoperation(self, space, message):
         w_exc = space.getattr(space.getbuiltinmodule('_io'),
-                              space.wrap('UnsupportedOperation'))
-        raise OperationError(w_exc, space.wrap(message))
+                              space.newtext('UnsupportedOperation'))
+        raise OperationError(w_exc, space.newtext(message))
 
     def _check_init(self, space):
         raise NotImplementedError
 
     def _deprecated_max_buffer_size(self, space):
-        space.warn(space.wrap("max_buffer_size is deprecated"),
+        space.warn(space.newtext("max_buffer_size is deprecated"),
                    space.w_DeprecationWarning)
 
     def read_w(self, space, w_size=None):
@@ -87,13 +87,13 @@
     def readinto_w(self, space, w_buffer):
         rwbuffer = space.getarg_w('w*', w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, "read", space.wrap(length))
+        w_data = space.call_method(self, "read", space.newint(length))
 
         if not space.isinstance_w(w_data, space.w_str):
             raise oefmt(space.w_TypeError, "read() should return bytes")
         data = space.str_w(w_data)
         rwbuffer.setslice(0, data)
-        return space.wrap(len(data))
+        return space.newint(len(data))
 
 W_BufferedIOBase.typedef = TypeDef(
     '_io._BufferedIOBase', W_IOBase.typedef,
@@ -190,15 +190,15 @@
 
     def closed_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("closed"))
+        return space.getattr(self.w_raw, space.newtext("closed"))
 
     def name_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("name"))
+        return space.getattr(self.w_raw, space.newtext("name"))
 
     def mode_get_w(self, space):
         self._check_init(space)
-        return space.getattr(self.w_raw, space.wrap("mode"))
+        return space.getattr(self.w_raw, space.newtext("mode"))
 
     def readable_w(self, space):
         self._check_init(space)
@@ -219,14 +219,14 @@
     def repr_w(self, space):
         typename = space.type(self).name
         try:
-            w_name = space.getattr(self, space.wrap("name"))
+            w_name = space.getattr(self, space.newtext("name"))
         except OperationError as e:
             if not e.match(space, space.w_Exception):
                 raise
-            return space.wrap("<%s>" % (typename,))
+            return space.newtext("<%s>" % (typename,))
         else:
             name_repr = space.str_w(space.repr(w_name))
-            return space.wrap("<%s name=%s>" % (typename, name_repr))
+            return space.newtext("<%s name=%s>" % (typename, name_repr))
 
     # ______________________________________________
 
@@ -247,7 +247,7 @@
     def tell_w(self, space):
         self._check_init(space)
         pos = self._raw_tell(space) - self._raw_offset()
-        return space.wrap(pos)
+        return space.newint(pos)
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
@@ -273,7 +273,7 @@
                     newpos = self.pos + offset
                     assert newpos >= 0
                     self.pos = newpos
-                    return space.wrap(current - available + offset)
+                    return space.newint(current - available + offset)
 
         # Fallback: invoke raw seek() method and clear buffer
         with self.lock:
@@ -287,11 +287,11 @@
             self.raw_pos = -1
             if self.readable:
                 self._reader_reset_buf()
-            return space.wrap(n)
+            return space.newint(n)
 
     def _raw_seek(self, space, pos, whence):
         w_pos = space.call_method(self.w_raw, "seek",
-                                  space.wrap(pos), space.wrap(whence))
+                                  space.newint(pos), space.newint(whence))
         pos = space.r_longlong_w(w_pos)
         if pos < 0:
             raise oefmt(space.w_IOError,
@@ -300,7 +300,7 @@
         return pos
 
     def _closed(self, space):
-        return space.is_true(space.getattr(self.w_raw, space.wrap("closed")))
+        return space.is_true(space.getattr(self.w_raw, space.newtext("closed")))
 
     def close_w(self, space):
         self._check_init(space)
@@ -723,7 +723,7 @@
                 self._adjust_position(self.pos + size)
                 if self.pos > self.write_end:
                     self.write_end = self.pos
-                return space.wrap(size)
+                return space.newint(size)
 
             # First write the current buffer
             try:
@@ -753,7 +753,7 @@
                         self.buffer[self.write_end + i] = data[i]
                     self.write_end += size
                     self.pos += size
-                    return space.wrap(size)
+                    return space.newint(size)
                 # Buffer as much as possible
                 for i in range(available):
                     self.buffer[self.write_end + i] = data[i]
@@ -809,7 +809,7 @@
             self.write_end = remaining
             self._adjust_position(remaining)
             self.raw_pos = 0
-        return space.wrap(written)
+        return space.newint(written)
 
     def flush_w(self, space):
         self._check_init(space)
@@ -921,13 +921,13 @@
             if self.w_writer is None:
                 raise oefmt(space.w_ValueError,
                             "I/O operation on uninitialized object")
-            w_meth = space.getattr(self.w_writer, space.wrap(method))
+            w_meth = space.getattr(self.w_writer, space.newtext(method))
             w_result = space.call_args(w_meth, __args__)
         if reader:
             if self.w_reader is None:
                 raise oefmt(space.w_ValueError,
                             "I/O operation on uninitialized object")
-            w_meth = space.getattr(self.w_reader, space.wrap(method))
+            w_meth = space.getattr(self.w_reader, space.newtext(method))
             w_result = space.call_args(w_meth, __args__)
         return w_result
     return method_w
@@ -977,7 +977,7 @@
         if self.w_writer is None:
             raise oefmt(space.w_ValueError,
                         "I/O operation on uninitialized object")
-        w_meth = space.getattr(self.w_writer, space.wrap("close"))
+        w_meth = space.getattr(self.w_writer, space.newtext("close"))
         try:
             space.call_args(w_meth, __args__)
         except OperationError as e:
@@ -988,7 +988,7 @@
         if self.w_reader is None:
             raise oefmt(space.w_ValueError,
                         "I/O operation on uninitialized object")
-        w_meth = space.getattr(self.w_reader, space.wrap("close"))
+        w_meth = space.getattr(self.w_reader, space.newtext("close"))
         space.call_args(w_meth, __args__)
 
         if e:
@@ -1000,7 +1000,7 @@
         return space.call_method(self.w_reader, "isatty")
 
     def closed_get_w(self, space):
-        return space.getattr(self.w_writer, space.wrap("closed"))
+        return space.getattr(self.w_writer, space.newtext("closed"))
 
 methods = dict((method, interp2app(getattr(W_BufferedRWPair, method + '_w')))
                for method in ['read', 'peek', 'read1', 'readinto', 'readable',
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -27,7 +27,7 @@
         if self.is_closed():
             if message is None:
                 message = "I/O operation on closed file"
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -49,16 +49,16 @@
 
         output = self.read(size)
         rwbuffer.setslice(0, output)
-        return space.wrap(len(output))
+        return space.newint(len(output))
 
     def write_w(self, space, w_data):
         self._check_closed(space)
         buf = space.buffer_w(w_data, space.BUF_CONTIG_RO).as_str()
         length = len(buf)
         if length <= 0:
-            return space.wrap(0)
+            return space.newint(0)
         self.write(buf)
-        return space.wrap(length)
+        return space.newint(length)
 
     def truncate_w(self, space, w_size=None):
         self._check_closed(space)
@@ -77,7 +77,7 @@
             self.seek(0, 2)
         else:
             self.seek(pos)
-        return space.wrap(size)
+        return space.newint(size)
 
     def getvalue_w(self, space):
         self._check_closed(space)
@@ -85,7 +85,7 @@
 
     def tell_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.tell())
+        return space.newint(self.tell())
 
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
@@ -105,7 +105,7 @@
                         "whence must be between 0 and 2, not %d", whence)
 
         self.seek(pos, whence)
-        return space.wrap(self.tell())
+        return space.newint(self.tell())
 
     def readable_w(self, space):
         self._check_closed(space)
@@ -123,13 +123,13 @@
         self.close()
 
     def closed_get_w(self, space):
-        return space.wrap(self.is_closed())
+        return space.newbool(self.is_closed())
 
     def getstate_w(self, space):
         self._check_closed(space)
         return space.newtuple([
             space.newbytes(self.getvalue()),
-            space.wrap(self.tell()),
+            space.newint(self.tell()),
             self.getdict(space)])
 
     def setstate_w(self, space, w_state):
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -13,7 +13,7 @@
     def fget(space, obj):
         w_value = getattr(obj, name)
         if w_value is None:
-            raise OperationError(space.w_AttributeError, space.wrap(name))
+            raise OperationError(space.w_AttributeError, space.newtext(name))
         else:
             return w_value
     def fset(space, obj, w_value):
@@ -21,7 +21,7 @@
     def fdel(space, obj):
         w_value = getattr(obj, name)
         if w_value is None:
-            raise OperationError(space.w_AttributeError, space.wrap(name))
+            raise OperationError(space.w_AttributeError, space.newtext(name))
         setattr(obj, name, None)
 
     return GetSetProperty(fget, fset, fdel, cls=cls, doc=doc)
@@ -126,7 +126,7 @@
     def descr_new(space, w_subtype, __args__):
         self = space.allocate_instance(W_FileIO, w_subtype)
         W_FileIO.__init__(self, space)
-        return space.wrap(self)
+        return self
 
     @unwrap_spec(mode=str, closefd=int)
     def descr_init(self, space, w_name, mode='r', closefd=True):
@@ -174,7 +174,7 @@
                     fd_is_own = True
 
             self._dircheck(space, w_name)
-            space.setattr(self, space.wrap("name"), w_name)
+            space.setattr(self, space.newtext("name"), w_name)
 
             if self.appending:
                 # For consistent behaviour, we explicitly seek to the end of file
@@ -203,7 +203,7 @@
             return 'wb'
 
     def descr_get_mode(self, space):
-        return space.wrap(self._mode())
+        return space.newtext(self._mode())
 
     def _closed(self, space):
         return self.fd < 0
@@ -212,7 +212,7 @@
         if message is None:
             message = "I/O operation on closed file"
         if self.fd < 0:
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def _check_readable(self, space):
         if not self.readable:
@@ -270,7 +270,7 @@
         except OSError as e:
             raise wrap_oserror(space, e,
                                exception_name='w_IOError')
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def tell_w(self, space):
         self._check_closed(space)
@@ -279,15 +279,15 @@
         except OSError as e:
             raise wrap_oserror(space, e,
                                exception_name='w_IOError')
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def readable_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.readable)
+        return space.newbool(self.readable)
 
     def writable_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.writable)
+        return space.newbool(self.writable)
 
     def seekable_w(self, space):
         self._check_closed(space)
@@ -304,7 +304,7 @@
 
     def fileno_w(self, space):
         self._check_closed(space)
-        return space.wrap(self.fd)
+        return space.newint(self.fd)
 
     def isatty_w(self, space):
         self._check_closed(space)
@@ -312,19 +312,19 @@
             res = os.isatty(self.fd)
         except OSError as e:
             raise wrap_oserror(space, e, exception_name='w_IOError')
-        return space.wrap(res)
+        return space.newbool(res)
 
     def repr_w(self, space):
         if self.fd < 0:
-            return space.wrap("<_io.FileIO [closed]>")
+            return space.newtext("<_io.FileIO [closed]>")
 
         if self.w_name is None:
-            return space.wrap(
+            return space.newtext(
                 "<_io.FileIO fd=%d mode='%s'>" % (
                     self.fd, self._mode()))
         else:
             w_repr = space.repr(self.w_name)
-            return space.wrap(
+            return space.newtext(
                 "<_io.FileIO name=%s mode='%s'>" % (
                     space.str_w(w_repr), self._mode()))
 
@@ -343,7 +343,7 @@
             raise wrap_oserror(space, e,
                                exception_name='w_IOError')
 
-        return space.wrap(n)
+        return space.newint(n)
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -376,7 +376,7 @@
             raise wrap_oserror(space, e,
                                exception_name='w_IOError')
         rwbuffer.setslice(0, buf)
-        return space.wrap(len(buf))
+        return space.newint(len(buf))
 
     def readall_w(self, space):
         self._check_closed(space)
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -103,7 +103,7 @@
         raise oefmt(space.w_ValueError,
                     "binary mode doesn't take a newline argument")
     w_raw = space.call_function(
-        space.gettypefor(W_FileIO), w_file, space.wrap(rawmode), space.wrap(closefd)
+        space.gettypefor(W_FileIO), w_file, space.newtext(rawmode), space.newbool(closefd)
     )
 
     isatty = space.is_true(space.call_method(w_raw, "isatty"))
@@ -142,17 +142,17 @@
     else:
         raise oefmt(space.w_ValueError, "unknown mode: '%s'", mode)
     w_buffer = space.call_function(
-        space.gettypefor(buffer_cls), w_raw, space.wrap(buffering)
+        space.gettypefor(buffer_cls), w_raw, space.newint(buffering)
     )
     if binary:
         return w_buffer
 
     w_wrapper = space.call_function(space.gettypefor(W_TextIOWrapper),
         w_buffer,
-        space.wrap(encoding),
-        space.wrap(errors),
-        space.wrap(newline),
-        space.wrap(line_buffering)
+        space.newtext_or_none(encoding),
+        space.newtext_or_none(errors),
+        space.newtext_or_none(newline),
+        space.newbool(line_buffering)
     )
-    space.setattr(w_wrapper, space.wrap("mode"), space.wrap(mode))
+    space.setattr(w_wrapper, space.newtext("mode"), space.newtext(mode))
     return w_wrapper
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -23,15 +23,15 @@
         return False
     try:
         w_value = error.get_w_value(space)
-        w_errno = space.getattr(w_value, space.wrap("errno"))
-        return space.eq_w(w_errno, space.wrap(EINTR))
+        w_errno = space.getattr(w_value, space.newtext("errno"))
+        return space.eq_w(w_errno, space.newint(EINTR))
     except OperationError:
         return False
 
 def unsupported(space, message):
     w_exc = space.getattr(space.getbuiltinmodule('_io'),
-                          space.wrap('UnsupportedOperation'))
-    return OperationError(w_exc, space.wrap(message))
+                          space.newtext('UnsupportedOperation'))
+    return OperationError(w_exc, space.newtext(message))
 
 # May be called with any object
 def check_readable_w(space, w_obj):
@@ -68,14 +68,14 @@
     def _closed(self, space):
         # This gets the derived attribute, which is *not* __IOBase_closed
         # in most cases!
-        w_closed = space.findattr(self, space.wrap('closed'))
+        w_closed = space.findattr(self, space.newtext('closed'))
         if w_closed is not None and space.is_true(w_closed):
             return True
         return False
 
     def _finalize_(self):
         space = self.space
-        w_closed = space.findattr(self, space.wrap('closed'))
+        w_closed = space.findattr(self, space.newtext('closed'))
         try:
             # If `closed` doesn't exist or can't be evaluated as bool, then
             # the object is probably in an unusable state, so ignore.
@@ -102,7 +102,7 @@
             message = "I/O operation on closed file"
         if self._closed(space):
             raise OperationError(
-                space.w_ValueError, space.wrap(message))
+                space.w_ValueError, space.newtext(message))
 
     def check_closed_w(self, space):
         self._check_closed(space)
@@ -126,7 +126,7 @@
         self._unsupportedoperation(space, "seek")
 
     def tell_w(self, space):
-        return space.call_method(self, "seek", space.wrap(0), space.wrap(1))
+        return space.call_method(self, "seek", space.newint(0), space.newint(1))
 
     def truncate_w(self, space, w_size=None):
         self._unsupportedoperation(space, "truncate")
@@ -136,14 +136,14 @@
 
     def enter_w(self, space):
         self._check_closed(space)
-        return space.wrap(self)
+        return self
 
     def exit_w(self, space, __args__):
         space.call_method(self, "close")
 
     def iter_w(self, space):
         self._check_closed(space)
-        return space.wrap(self)
+        return self
 
     def next_w(self, space):
         w_line = space.call_method(self, "readline")
@@ -170,7 +170,7 @@
         # For backwards compatibility, a (slowish) readline().
         limit = convert_size(space, w_limit)
 
-        has_peek = space.findattr(self, space.wrap("peek"))
+        has_peek = space.findattr(self, space.newtext("peek"))
 
         builder = StringBuilder()
         size = 0
@@ -180,7 +180,7 @@
 
             if has_peek:
                 try:
-                    w_readahead = space.call_method(self, "peek", space.wrap(1))
+                    w_readahead = space.call_method(self, "peek", space.newint(1))
                 except OperationError as e:
                     if trap_eintr(space, e):
                         continue
@@ -210,7 +210,7 @@
                     nreadahead = n
 
             try:
-                w_read = space.call_method(self, "read", space.wrap(nreadahead))
+                w_read = space.call_method(self, "read", space.newint(nreadahead))
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue
@@ -327,7 +327,7 @@
         while True:
             try:
                 w_data = space.call_method(self, "read",
-                                           space.wrap(DEFAULT_BUFFER_SIZE))
+                                           space.newint(DEFAULT_BUFFER_SIZE))
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -28,11 +28,11 @@
 
         if (newline is not None and newline != u"" and newline != u"\n" and
             newline != u"\r" and newline != u"\r\n"):
-            # Not using oefmt() because I don't know how to ues it
+            # Not using oefmt() because I don't know how to use it
             # with unicode
             raise OperationError(space.w_ValueError,
                 space.mod(
-                    space.wrap("illegal newline value: %s"), space.wrap(newline)
+                    space.newtext("illegal newline value: %s"), w_newline
                 )
             )
         if newline is not None:
@@ -45,7 +45,7 @@
             self.w_decoder = space.call_function(
                 space.gettypefor(W_IncrementalNewlineDecoder),
                 space.w_None,
-                space.wrap(int(self.readtranslate))
+                space.newint(int(self.readtranslate))
             )
 
         if not space.is_none(w_initvalue):
@@ -58,9 +58,9 @@
         if self.readnl is None:
             w_readnl = space.w_None
         else:
-            w_readnl = space.str(space.wrap(self.readnl))
+            w_readnl = space.str(space.newunicode(self.readnl)) # YYY
         return space.newtuple([
-            w_initialval, w_readnl, space.wrap(self.pos), w_dict
+            w_initialval, w_readnl, space.newint(self.pos), w_dict
         ])
 
     def descr_setstate(self, space, w_state):
@@ -105,7 +105,7 @@
         if self.buf is None:
             if message is None:
                 message = "I/O operation on closed file"
-            raise OperationError(space.w_ValueError, space.wrap(message))
+            raise OperationError(space.w_ValueError, space.newtext(message))
 
     def resize_buffer(self, newlength):
         if len(self.buf) > newlength:
@@ -139,7 +139,7 @@
 
         if self.writenl:
             w_decoded = space.call_method(
-                w_decoded, "replace", space.wrap("\n"), space.wrap(self.writenl)
+                w_decoded, "replace", space.newtext("\n"), space.newunicode(self.writenl)
             )
 
         string = space.unicode_w(w_decoded)
@@ -147,7 +147,7 @@
 
         if size:
             self.write(string)
-        return space.wrap(orig_size)
+        return space.newint(orig_size)
 
     def read_w(self, space, w_size=None):
         self._check_closed(space)
@@ -155,21 +155,21 @@
         start = self.pos
         available = len(self.buf) - start
         if available <= 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
         if size >= 0 and size <= available:
             end = start + size
         else:
             end = len(self.buf)
         assert 0 <= start <= end
         self.pos = end
-        return space.wrap(u''.join(self.buf[start:end]))
+        return space.newunicode(u''.join(self.buf[start:end]))
 
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
 
         if self.pos >= len(self.buf):
-            return space.wrap(u"")
+            return space.newunicode(u"")
 
         start = self.pos
         if limit < 0 or limit > len(self.buf) - self.pos:
@@ -190,7 +190,7 @@
             endpos = end
         assert endpos >= 0
         self.pos = endpos
-        return space.wrap(u"".join(self.buf[start:endpos]))
+        return space.newunicode(u"".join(self.buf[start:endpos]))
 
     @unwrap_spec(pos=int, mode=int)
     def seek_w(self, space, pos, mode=0):
@@ -212,7 +212,7 @@
 
         assert pos >= 0
         self.pos = pos
-        return space.wrap(pos)
+        return space.newint(pos)
 
     def truncate_w(self, space, w_size=None):
         self._check_closed(space)
@@ -227,11 +227,11 @@
         if size < len(self.buf):
             self.resize_buffer(size)
 
-        return space.wrap(size)
+        return space.newint(size)
 
     def getvalue_w(self, space):
         self._check_closed(space)
-        return space.wrap(u''.join(self.buf))
+        return space.newunicode(u''.join(self.buf))
 
     def readable_w(self, space):
         self._check_closed(space)
@@ -249,7 +249,7 @@
         self.buf = None
 
     def closed_get_w(self, space):
-        return space.wrap(self.buf is None)
+        return space.newbool(self.buf is None)
 
     def line_buffering_get_w(self, space):
         return space.w_False
@@ -257,7 +257,7 @@
     def newlines_get_w(self, space):
         if self.w_decoder is None:
             return space.w_None
-        return space.getattr(self.w_decoder, space.wrap("newlines"))
+        return space.getattr(self.w_decoder, space.newtext("newlines"))
 
 
 W_StringIO.typedef = TypeDef(
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -29,17 +29,17 @@
 
     def __init__(self, space):
         self.w_newlines_dict = {
-            SEEN_CR: space.wrap(u"\r"),
-            SEEN_LF: space.wrap(u"\n"),
-            SEEN_CRLF: space.wrap(u"\r\n"),
+            SEEN_CR: space.newunicode(u"\r"),
+            SEEN_LF: space.newunicode(u"\n"),
+            SEEN_CRLF: space.newunicode(u"\r\n"),
             SEEN_CR | SEEN_LF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\n")]),
             SEEN_CR | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\r\n")]),
             SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\n"), space.newunicode(u"\r\n")]),
             SEEN_CR | SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.newunicode(u"\r"), space.newunicode(u"\n"), space.newunicode(u"\r\n")]),
             }
 
     @unwrap_spec(translate=int)
@@ -47,7 +47,7 @@
         self.w_decoder = w_decoder
         self.translate = translate
         if space.is_none(w_errors):
-            self.w_errors = space.wrap("strict")
+            self.w_errors = space.newtext("strict")
         else:
             self.w_errors = w_errors
 
@@ -65,7 +65,7 @@
         # decode input (with the eventual \r from a previous pass)
         if not space.is_w(self.w_decoder, space.w_None):
             w_output = space.call_method(self.w_decoder, "decode",
-                                         w_input, space.wrap(final))
+                                         w_input, space.newbool(final))
         else:
             w_output = w_input
 
@@ -91,7 +91,7 @@
                 output_len -= 1
 
         if output_len == 0:
-            return space.wrap(u"")
+            return space.newunicode(u"")
 
         # Record which newlines are read and do newline translation if
         # desired, all in one pass.
@@ -146,7 +146,7 @@
             output = builder.build()
 
         self.seennl |= seennl
-        return space.wrap(output)
+        return space.newunicode(output)
 
     def reset_w(self, space):
         self.seennl = 0
@@ -165,7 +165,7 @@
         flag <<= 1
         if self.pendingcr:
             flag |= 1
-        return space.newtuple([w_buffer, space.wrap(flag)])
+        return space.newtuple([w_buffer, space.newint(flag)])
 
     def setstate_w(self, space, w_state):
         w_buffer, w_flag = space.unpackiterable(w_state, 2)
@@ -174,7 +174,7 @@
         flag >>= 1
 
         if self.w_decoder and not space.is_w(self.w_decoder, space.w_None):
-            w_state = space.newtuple([w_buffer, space.wrap(flag)])
+            w_state = space.newtuple([w_buffer, space.newint(flag)])
             space.call_method(self.w_decoder, "setstate", w_state)
 
 W_IncrementalNewlineDecoder.typedef = TypeDef(
@@ -272,17 +272,17 @@
 
 def _determine_encoding(space, encoding):
     if encoding is not None:
-        return space.wrap(encoding)
+        return space.newtext(encoding)
 
     try:
         w_locale = space.call_method(space.builtin, '__import__',
-                                     space.wrap('locale'))
+                                     space.newtext('locale'))
         w_encoding = space.call_method(w_locale, 'getpreferredencoding')
     except OperationError as e:
         # getpreferredencoding() may also raise ImportError
         if not e.match(space, space.w_ImportError):
             raise
-        return space.wrap('ascii')
+        return space.newtext('ascii')
     else:
         if space.isinstance_w(w_encoding, space.w_str):
             return w_encoding
@@ -367,7 +367,7 @@
         self.w_encoding = _determine_encoding(space, encoding)
 
         if space.is_none(w_errors):
-            w_errors = space.wrap("strict")
+            w_errors = space.newtext("strict")
         self.w_errors = w_errors
 
         if space.is_none(w_newline):
@@ -404,7 +404,7 @@
             if self.readuniversal:
                 self.w_decoder = space.call_function(
                     space.gettypeobject(W_IncrementalNewlineDecoder.typedef),
-                    self.w_decoder, space.wrap(self.readtranslate))
+                    self.w_decoder, space.newbool(self.readtranslate))
 
         # build the encoder object
         if space.is_true(space.call_method(w_buffer, "writable")):
@@ -420,9 +420,9 @@
         if self.seekable and self.w_encoder:
             self.encoding_start_of_stream = True
             w_cookie = space.call_method(self.w_buffer, "tell")
-            if not space.eq_w(w_cookie, space.wrap(0)):
+            if not space.eq_w(w_cookie, space.newint(0)):
                 self.encoding_start_of_stream = False
-                space.call_method(self.w_encoder, "setstate", space.wrap(0))
+                space.call_method(self.w_encoder, "setstate", space.newint(0))
 
         self.state = STATE_OK
 
@@ -443,14 +443,14 @@
 
     def descr_repr(self, space):
         self._check_init(space)
-        w_name = space.findattr(self, space.wrap("name"))
+        w_name = space.findattr(self, space.newtext("name"))
         if w_name is None:
-            w_name_str = space.wrap("")
+            w_name_str = space.newtext("")
         else:
-            w_name_str = space.mod(space.wrap("name=%r "), w_name)
+            w_name_str = space.mod(space.newtext("name=%r "), w_name)
         w_args = space.newtuple([w_name_str, self.w_encoding])
         return space.mod(
-            space.wrap("<_io.TextIOWrapper %sencoding=%r>"), w_args
+            space.newtext("<_io.TextIOWrapper %sencoding=%r>"), w_args
         )
 
     def readable_w(self, space):
@@ -475,17 +475,17 @@
 
     def closed_get_w(self, space):
         self._check_attached(space)
-        return space.getattr(self.w_buffer, space.wrap("closed"))
+        return space.getattr(self.w_buffer, space.newtext("closed"))
 
     def newlines_get_w(self, space):
         self._check_attached(space)
         if self.w_decoder is None:
             return space.w_None
-        return space.findattr(self.w_decoder, space.wrap("newlines"))
+        return space.findattr(self.w_decoder, space.newtext("newlines"))
 
     def name_get_w(self, space):
         self._check_attached(space)
-        return space.getattr(self.w_buffer, space.wrap("name"))
+        return space.getattr(self.w_buffer, space.newtext("name"))
 
     def flush_w(self, space):
         self._check_attached(space)
@@ -504,7 +504,7 @@
     def close_w(self, space):
         self._check_attached(space)
         if not space.is_true(space.getattr(self.w_buffer,
-                                           space.wrap("closed"))):
+                                           space.newtext("closed"))):
             try:
                 space.call_method(self, "flush")
             finally:
@@ -564,7 +564,7 @@
 
         # Read a chunk, decode it, and put the result in self._decoded_chars
         w_input = space.call_method(self.w_buffer, "read1",
-                                    space.wrap(self.chunk_size))
+                                    space.newint(self.chunk_size))
 
         if not space.isinstance_w(w_input, space.w_str):
             msg = "decoder getstate() should have returned a bytes " \
@@ -573,7 +573,7 @@
 
         eof = space.len_w(w_input) == 0
         w_decoded = space.call_method(self.w_decoder, "decode",
-                                      w_input, space.wrap(eof))
+                                      w_input, space.newbool(eof))
         check_decoded(space, w_decoded)
         self._set_decoded_chars(space.unicode_w(w_decoded))
         if space.len_w(w_decoded) > 0:
@@ -611,7 +611,7 @@
             w_bytes = space.call_method(self.w_buffer, "read")
             w_decoded = space.call_method(self.w_decoder, "decode", w_bytes, space.w_True)
             check_decoded(space, w_decoded)
-            w_result = space.wrap(self._get_decoded_chars(-1))
+            w_result = space.newunicode(self._get_decoded_chars(-1))
             w_final = space.add(w_result, w_decoded)
             self.snapshot = None
             return w_final
@@ -637,7 +637,7 @@
                     continue
                 raise
 
-        return space.wrap(builder.build())
+        return space.newunicode(builder.build())
 
     def readline_w(self, space, w_limit=None):
         self._check_attached(space)
@@ -728,9 +728,9 @@
             line = u''.join(chunks)
 
         if line:
-            return space.wrap(line)
+            return space.newunicode(line)
         else:
-            return space.wrap(u'')
+            return space.newunicode(u'')
 
     # _____________________________________________________________
     # write methods
@@ -754,8 +754,8 @@
             if text.find(u'\n') >= 0:
                 haslf = True
         if haslf and self.writetranslate and self.writenl:
-            w_text = space.call_method(w_text, "replace", space.wrap(u'\n'),
-                                       space.wrap(self.writenl))
+            w_text = space.call_method(w_text, "replace", space.newunicode(u'\n'),
+                                       space.newunicode(self.writenl))
             text = space.unicode_w(w_text)
 
         needflush = False
@@ -787,7 +787,7 @@
         if self.w_decoder:
             space.call_method(self.w_decoder, "reset")
 
-        return space.wrap(textlen)
+        return space.newint(textlen)
 
     def _writeflush(self, space):
         if not self.pending_bytes:
@@ -830,14 +830,14 @@
         else:
             space.call_method(self.w_decoder, "setstate",
                               space.newtuple([space.newbytes(""),
-                                              space.wrap(cookie.dec_flags)]))
+                                              space.newint(cookie.dec_flags)]))
 
     def _encoder_setstate(self, space, cookie):
         if cookie.start_pos == 0 and cookie.dec_flags == 0:
             space.call_method(self.w_encoder, "reset")
             self.encoding_start_of_stream = True
         else:
-            space.call_method(self.w_encoder, "setstate", space.wrap(0))
+            space.call_method(self.w_encoder, "setstate", space.newint(0))
             self.encoding_start_of_stream = False
 
     @unwrap_spec(whence=int)
@@ -849,7 +849,7 @@
 
         if whence == 1:
             # seek relative to current position
-            if not space.is_true(space.eq(w_pos, space.wrap(0))):
+            if not space.is_true(space.eq(w_pos, space.newint(0))):
                 raise oefmt(space.w_IOError,
                             "can't do nonzero cur-relative seeks")
             # Seeking to the current position should attempt to sync the
@@ -858,7 +858,7 @@
 
         elif whence == 2:
             # seek relative to end of file
-            if not space.is_true(space.eq(w_pos, space.wrap(0))):
+            if not space.is_true(space.eq(w_pos, space.newint(0))):
                 raise oefmt(space.w_IOError,
                             "can't do nonzero end-relative seeks")
             space.call_method(self, "flush")
@@ -867,14 +867,14 @@
             if self.w_decoder:
                 space.call_method(self.w_decoder, "reset")
             return space.call_method(self.w_buffer, "seek",
-                                     w_pos, space.wrap(whence))
+                                     w_pos, space.newint(whence))
 
         elif whence != 0:
             raise oefmt(space.w_ValueError,
                         "invalid whence (%d, should be 0, 1 or 2)",
                         whence)
 
-        if space.is_true(space.lt(w_pos, space.wrap(0))):
+        if space.is_true(space.lt(w_pos, space.newint(0))):
             r = space.str_w(space.repr(w_pos))
             raise oefmt(space.w_ValueError,
                         "negative seek position %s", r)
@@ -886,7 +886,7 @@
         cookie = PositionCookie(space.bigint_w(w_pos))
 
         # Seek back to the safe start point
-        space.call_method(self.w_buffer, "seek", space.wrap(cookie.start_pos))
+        space.call_method(self.w_buffer, "seek", space.newint(cookie.start_pos))
 
         self._set_decoded_chars(None)
         self.snapshot = None
@@ -898,7 +898,7 @@
         if cookie.chars_to_skip:
             # Just like _read_chunk, feed the decoder and save a snapshot.
             w_chunk = space.call_method(self.w_buffer, "read",
-                                        space.wrap(cookie.bytes_to_feed))
+                                        space.newint(cookie.bytes_to_feed))
             if not space.isinstance_w(w_chunk, space.w_str):
                 msg = "underlying read() should have returned " \
                       "a bytes object, not '%T'"
@@ -908,7 +908,7 @@
                                              space.bytes_w(w_chunk))
 
             w_decoded = space.call_method(self.w_decoder, "decode",
-                                          w_chunk, space.wrap(cookie.need_eof))
+                                          w_chunk, space.newbool(cookie.need_eof))
             check_decoded(space, w_decoded)
             self._set_decoded_chars(space.unicode_w(w_decoded))
 
@@ -1000,8 +1000,8 @@
             else:
                 # We didn't get enough decoded data; signal EOF to get more.
                 w_decoded = space.call_method(self.w_decoder, "decode",
-                                              space.wrap(""),
-                                              space.wrap(1)) # final=1
+                                              space.newbytes(""),
+                                              space.newint(1)) # final=1
                 check_decoded(space, w_decoded)
                 chars_decoded += len(space.unicode_w(w_decoded))
                 cookie.need_eof = 1
@@ -1018,7 +1018,7 @@
 
     def chunk_size_get_w(self, space):
         self._check_attached(space)
-        return space.wrap(self.chunk_size)
+        return space.newint(self.chunk_size)
 
     def chunk_size_set_w(self, space, w_size):
         self._check_attached(space)

From pypy.commits at gmail.com  Mon Nov  7 08:53:48 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 07 Nov 2016 05:53:48 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: pyexpat
Message-ID: <5820876c.060ec20a.9d15b.d278@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88173:39100ec8a0e5
Date: 2016-11-07 14:53 +0100
http://bitbucket.org/pypy/pypy/changeset/39100ec8a0e5/

Log:	pyexpat

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -256,9 +256,10 @@
                 'w_arg%d = parser.w_convert_model(space, arg%d)' % (i, i))
             converters.append(
                 'XML_FreeContentModel(parser.itself, arg%d)' % (i,))
+        elif ARG == rffi.INT:
+            converters.append("w_arg%d = space.newint(arg%d)" % (i, i))
         else:
-            converters.append(
-                'w_arg%d = space.wrap(arg%d)' % (i, i))
+            assert 0, "missing conversion case"
         real_params.append(ARG)
     converters = '; '.join(converters)
 
@@ -408,13 +409,13 @@
     'XML_ExpatVersion', [], rffi.CCHARP)
 
 def get_expat_version(space):
-    return space.wrap(rffi.charp2str(XML_ExpatVersion()))
+    return space.newtext(rffi.charp2str(XML_ExpatVersion()))
 
 def get_expat_version_info(space):
     return space.newtuple([
-        space.wrap(XML_MAJOR_VERSION),
-        space.wrap(XML_MINOR_VERSION),
-        space.wrap(XML_MICRO_VERSION)])
+        space.newint(XML_MAJOR_VERSION),
+        space.newint(XML_MINOR_VERSION),
+        space.newint(XML_MICRO_VERSION)])
 
 class Cache:
     def __init__(self, space):
@@ -545,8 +546,8 @@
         children = [self.w_convert_model(space, model.c_children[i])
                     for i in range(model.c_numchildren)]
         return space.newtuple([
-            space.wrap(model.c_type),
-            space.wrap(model.c_quant),
+            space.newint(model.c_type),
+            space.newint(model.c_quant),
             self.w_convert_charp(space, model.c_name),
             space.newtuple(children)])
 

From pypy.commits at gmail.com  Mon Nov  7 09:17:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 06:17:56 -0800 (PST)
Subject: [pypy-commit] cffi default: Reorganize (thanks matti)
Message-ID: <58208d14.46bb1c0a.33fda.5732@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2806:dc390236b9c6
Date: 2016-11-07 15:17 +0100
http://bitbucket.org/cffi/cffi/changeset/dc390236b9c6/

Log:	Reorganize (thanks matti)

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -6,13 +6,10 @@
 code from Python, based on C-like declarations that you can often
 copy-paste from header files or documentation.
 
-* Goals_
-
-  * `Comments and bugs`_
-
 .. toctree::
    :maxdepth: 2
 
+   index
    whatsnew
    installation
    overview

From pypy.commits at gmail.com  Mon Nov  7 09:20:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 06:20:15 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: Print another
 interp-level traceback here, because this place normally
Message-ID: <58208d9f.045d1c0a.b23cf.2a9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88174:6a5fc03446f2
Date: 2016-11-07 15:19 +0100
http://bitbucket.org/pypy/pypy/changeset/6a5fc03446f2/

Log:	Print another interp-level traceback here, because this place
	normally eats the rest of the interp-level traceback

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -712,6 +712,8 @@
             if not we_are_translated():
                 raise
             raise e
+        except OperationError:
+            raise
         except KeyboardInterrupt:
             raise OperationError(space.w_KeyboardInterrupt, space.w_None)
         except MemoryError:
@@ -722,6 +724,14 @@
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
+        except Exception as e:      # general fall-back
+            if we_are_translated():
+                from rpython.rlib.debug import debug_print_traceback
+                debug_print_traceback()
+            # propagate the exception anyway, which will be turned
+            # into a proper OperationError(SystemError) when we
+            # reach PyFrame.execute_frame()
+            raise
 
 # (verbose) performance hack below
 

From pypy.commits at gmail.com  Mon Nov  7 09:22:27 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 06:22:27 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak the upcoming release notes
Message-ID: <58208e23.0209c20a.36aab.e131@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88175:f57313e9eb2f
Date: 2016-11-07 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/f57313e9eb2f/

Log:	tweak the upcoming release notes

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -19,7 +19,8 @@
 We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
 well as maintaining support for previous versions.
 
-XXX
+CFFI_ has been updated to 1.9, improving an already great package for
+interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -38,7 +39,7 @@
 with making RPython's JIT even better.
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
-.. _cffi: https://cffi.readthedocs.org
+.. _CFFI: https://cffi.readthedocs.io/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
@@ -81,7 +82,7 @@
   * Support more attributes on ``super``
   * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
   * Restore the ability to translate with CPython
-  * Update to CFFI 1.8.4
+  * Update to CFFI 1.9.0
   * Support the new buffer protocol in cpyext and numpypy
   * Add ``rposix.sync()``
   * Support full-precision nanosecond times in os.stat()

From pypy.commits at gmail.com  Mon Nov  7 09:28:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 06:28:22 -0800 (PST)
Subject: [pypy-commit] cffi default: Backed out changeset dc390236b9c6
Message-ID: <58208f86.e8edc20a.1018a.e2df@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2807:59da78268d39
Date: 2016-11-07 15:28 +0100
http://bitbucket.org/cffi/cffi/changeset/59da78268d39/

Log:	Backed out changeset dc390236b9c6

	Sphynx bug (infinite recursion)

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -6,10 +6,13 @@
 code from Python, based on C-like declarations that you can often
 copy-paste from header files or documentation.
 
+* Goals_
+
+  * `Comments and bugs`_
+
 .. toctree::
    :maxdepth: 2
 
-   index
    whatsnew
    installation
    overview

From pypy.commits at gmail.com  Mon Nov  7 09:39:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 06:39:59 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: Emphasis potential
 plural
Message-ID: <5820923f.a8c8c20a.b63e.e95c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88176:0b3bcb8cab04
Date: 2016-11-07 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/0b3bcb8cab04/

Log:	Emphasis potential plural

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -907,7 +907,7 @@
         if we_are_translated():
             from rpython.rlib.debug import debug_print_traceback
             debug_print_traceback()
-            extra = '; internal traceback was dumped to stderr'
+            extra = '; internal traceback(s) were dumped to stderr'
         else:
             extra = self._convert_unexpected_exception_extra(e)
         operr = OperationError(self.space.w_SystemError, self.space.wrap(

From pypy.commits at gmail.com  Mon Nov  7 09:43:22 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 06:43:22 -0800 (PST)
Subject: [pypy-commit] pypy default: tweaks, formatting
Message-ID: <5820930a.4251c20a.7b642.ea3b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88177:9b53ab38c1ae
Date: 2016-11-07 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/9b53ab38c1ae/

Log:	tweaks, formatting

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -74,6 +74,7 @@
 =========================================================
 
 * New features
+
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
   * Backport fixes to rposix on windows from py3.5
@@ -88,7 +89,7 @@
   * Support full-precision nanosecond times in os.stat()
   * Add documentation about the assembler backends to RPYthon
   * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
-    changes downstream packaging requirments
+    changes downstream packaging requirements
   * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
   * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
     be ``str`` or ``unicode``)
@@ -106,6 +107,7 @@
   * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
+
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
   * Fix translation of the sandbox
   * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
@@ -126,6 +128,7 @@
   * ``_numpypy.add.reduce`` returns a scalar now
 
 * Performance improvements:
+
   * Improve method calls on oldstyle classes
   * Clean and refactor code for testing cpyext to allow sharing with py3.5
   * Refactor a building the map of reflected ops in ``_numpypy``
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.6.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.4+
-==========================
+=========================
+What's new in PyPy2.7 5.6
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc

From pypy.commits at gmail.com  Mon Nov  7 10:10:41 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 07:10:41 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak and reorder howto release
Message-ID: <58209971.e39cc20a.e2b77.e94d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88178:c563dd1b2750
Date: 2016-11-07 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c563dd1b2750/

Log:	tweak and reorder howto release

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -1,4 +1,4 @@
-The PyPy Release Process
+PyPy's Release Process
 ========================
 
 Release Policy
@@ -39,41 +39,60 @@
   Some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
   necessary.
-* update pypy/doc/contributor.rst (and possibly LICENSE)
-  pypy/doc/tool/makecontributor.py generates the list of contributors
-* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  create a fresh whatsnew_head.rst after the release
-  and add the new file to  pypy/doc/index-of-whatsnew.rst
-* go to pypy/tool/release and run
-  ``force-builds.py <release branch>``
-  The following JIT binaries should be built, however, we need more buildbots
-  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
-  freebsd64 
 
-* wait for builds to complete, make sure there are no failures
-* download the builds, repackage binaries. Tag the release version
-  and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+* Update and write documentation
 
-  Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
-  and to cobra, as some packagers prefer a clearly labeled source package
-  ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
-  unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+  * update pypy/doc/contributor.rst (and possibly LICENSE)
+    pypy/doc/tool/makecontributor.py generates the list of contributors
 
-* Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+  * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+    create a fresh whatsnew_head.rst after the release
+    and add the new file to  pypy/doc/index-of-whatsnew.rst
 
-* write release announcement pypy/doc/release-x.y(.z).txt
+  * write release announcement pypy/doc/release-VERSION.rst
+    The release announcement should contain a direct link to the download page
 
-  The release announcement should contain a direct link to the download page
+  * Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
 
-* Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
+* Build and upload the release tar-balls
 
-* update pypy.org (under extradoc/pypy.org), rebuild and commit
+  * go to pypy/tool/release and run
+    ``force-builds.py <release branch>``
+    The following JIT binaries should be built, however, we need more buildbots
+    windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+    freebsd64 
 
-* post announcement on morepypy.blogspot.com
-* send announcements to twitter.com, pypy-dev, python-list,
-  python-announce, python-dev ...
+  * wait for builds to complete, make sure there are no failures
 
-* add a tag on the pypy/jitviewer repo that corresponds to pypy release
-* add a tag on the codespeed web site that corresponds to pypy release
-* revise versioning at https://readthedocs.org/projects/pypy
+  * send out a mailing list message asking for people to test before uploading
+    to prevent having to upload more than once
+
+  * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
+    that the source tarball can be produced in the next steps
+
+  * download the builds, repackage binaries. Tag the release version
+    and download and repackage source from bitbucket. You may find it
+    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+
+    Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
+    and to cobra, as some packagers prefer a clearly labeled source package
+    ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
+    unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
+  * Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+
+* Send out a mailing list message asking for last-minute comments and testing
+
+* RELEASE !  
+
+  * update pypy.org (under extradoc/pypy.org), rebuild and commit, using the
+    hashes produced from the ``repackage.sh`` script or by hand
+
+  * post announcement on morepypy.blogspot.com
+  * send announcements to twitter.com, pypy-dev, python-list,
+    python-announce, python-dev ...
+
+* If all is OK, document the released version
+
+  * add a tag on the codespeed web site that corresponds to pypy release
+  * revise versioning at https://readthedocs.org/projects/pypy

From pypy.commits at gmail.com  Mon Nov  7 10:33:58 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 07:33:58 -0800 (PST)
Subject: [pypy-commit] pypy default: spelling (ronan's ghost)
Message-ID: <58209ee6.4a0e1c0a.b11ef.4be6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88179:4416b486384e
Date: 2016-11-07 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4416b486384e/

Log:	spelling (ronan's ghost)

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -6,7 +6,7 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
 
 We continue to make incremental improvements to our C-API
-compatability layer (cpyext). We pass all but a few of the tests in the
+compatibility layer (cpyext). We pass all but a few of the tests in the
 upstream numpy `test suite`_. 
 
 Work proceeds at a good pace on the PyPy3.5

From pypy.commits at gmail.com  Mon Nov  7 10:39:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 07:39:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix link
Message-ID: <5820a02b.058a1c0a.e5067.4769@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88180:cd69e6a938e5
Date: 2016-11-07 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/cd69e6a938e5/

Log:	Fix link

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -39,7 +39,7 @@
 with making RPython's JIT even better.
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
-.. _CFFI: https://cffi.readthedocs.io/latest/whatsnew.html
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org

From pypy.commits at gmail.com  Mon Nov  7 10:46:20 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 07:46:20 -0800 (PST)
Subject: [pypy-commit] pypy default: typo (Tiberium)
Message-ID: <5820a1cc.d39a1c0a.435b2.e0a8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88181:3b5d41df3e1f
Date: 2016-11-07 17:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3b5d41df3e1f/

Log:	typo (Tiberium)

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -96,7 +96,7 @@
   * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
   * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
     relevant platforms
-  * Backport the ``'faulthandler`` module from py3.5
+  * Backport the ``faulthandler`` module from py3.5
   * Improve the error message when trying to call a method where the ``self``
     parameter is missing in the definition
   * Implement ``rposix.cpu_count``

From pypy.commits at gmail.com  Mon Nov  7 10:48:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 07:48:24 -0800 (PST)
Subject: [pypy-commit] pypy rpython-error-to-systemerror: close branch,
 ready to merge
Message-ID: <5820a248.c24bc20a.3a44a.0501@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: rpython-error-to-systemerror
Changeset: r88182:1cb372e0ed0f
Date: 2016-11-07 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/1cb372e0ed0f/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Mon Nov  7 10:48:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 07:48:26 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge rpython-error-to-systemerror
Message-ID: <5820a24a.d5901c0a.24eae.49a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88183:b05f65fe2cc4
Date: 2016-11-07 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/b05f65fe2cc4/

Log:	hg merge rpython-error-to-systemerror

	Any uncaught RPython exception is turned into an app-level
	SystemError. The RPython traceback is also printed, but only up to
	the first Python frame. The rest is normally printed as a regular
	app-level traceback for the SystemError. This should improve a lot
	the live of users hitting an uncaught RPython error.

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -58,10 +58,14 @@
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
         s = self._w_value
-        if self.__class__ is not OperationError and s is None:
-            space = getattr(self.w_type, 'space')
-            if space is not None:
+        space = getattr(self.w_type, 'space', None)
+        if space is not None:
+            if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
+            try:
+                s = space.str_w(s)
+            except Exception:
+                pass
         return '[%s: %s]' % (self.w_type, s)
 
     def errorstr(self, space, use_repr=False):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -712,6 +712,8 @@
             if not we_are_translated():
                 raise
             raise e
+        except OperationError:
+            raise
         except KeyboardInterrupt:
             raise OperationError(space.w_KeyboardInterrupt, space.w_None)
         except MemoryError:
@@ -722,6 +724,14 @@
                         "maximum recursion depth exceeded")
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
+        except Exception as e:      # general fall-back
+            if we_are_translated():
+                from rpython.rlib.debug import debug_print_traceback
+                debug_print_traceback()
+            # propagate the exception anyway, which will be turned
+            # into a proper OperationError(SystemError) when we
+            # reach PyFrame.execute_frame()
+            raise
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,6 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+import sys
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
@@ -276,8 +277,13 @@
                     next_instr = r_uint(self.last_instr + 1)
                     if next_instr != 0:
                         self.pushvalue(w_inputvalue)
-                w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                            executioncontext)
+                try:
+                    w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                                executioncontext)
+                except OperationError:
+                    raise
+                except Exception as e:      # general fall-back
+                    raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
@@ -883,6 +889,34 @@
             frame = frame.f_backref()
         return None
 
+    def _convert_unexpected_exception_extra(self, e):
+        "NOT_RPYTHON"
+        if e.__class__.__name__ in (
+            'Skipped',     # list of exception class names that are ok
+            ):             # to get during ==untranslated tests== only
+            raise
+        # include the RPython-level traceback
+        exc = sys.exc_info()
+        import traceback, cStringIO
+        f = cStringIO.StringIO()
+        print >> f, "\nTraceback (interpreter-level):"
+        traceback.print_tb(exc[2], file=f)
+        return f.getvalue()
+
+    def _convert_unexpected_exception(self, e):
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback(s) were dumped to stderr'
+        else:
+            extra = self._convert_unexpected_exception_extra(e)
+        operr = OperationError(self.space.w_SystemError, self.space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
+        pytraceback.record_application_traceback(
+            self.space, operr, self, self.last_instr)
+        raise operr
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -1,8 +1,10 @@
 import py
 from pypy.interpreter import executioncontext
+from pypy.interpreter.error import OperationError
 
-class Finished(Exception):
-    pass
+class Finished(OperationError):
+    def __init__(self):
+        OperationError.__init__(self, "exception_class", "exception_value")
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -4,6 +4,7 @@
 from pypy.interpreter import gateway, argument
 from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.error import OperationError
 import py
 import sys
 
@@ -771,6 +772,21 @@
         w_g = space.wrap(gateway.interp2app_temp(g, doc='bar'))
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
+    def test_system_error(self):
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        e = py.test.raises(OperationError, space.appexec, [w_g], """(my_g):
+            my_g()
+        """)
+        err = str(e.value)
+        assert 'SystemError' in err
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -86,7 +86,9 @@
         return space.w_None
     return space.get(w_descr, w_obj)
 
-def do_what_I_mean(space):
+def do_what_I_mean(space, w_crash=None):
+    if not space.is_none(w_crash):
+        raise ValueError    # RPython-level, uncaught
     return space.wrap(42)
 
 
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -92,6 +92,7 @@
         from __pypy__ import do_what_I_mean
         x = do_what_I_mean()
         assert x == 42
+        raises(SystemError, do_what_I_mean, 1)
 
     def test_list_strategy(self):
         from __pypy__ import strategy
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -11,7 +11,7 @@
 
 # Expose these here (public interface)
 from rpython.rtyper.debug import (
-    ll_assert, FatalError, fatalerror, fatalerror_notb)
+    ll_assert, FatalError, fatalerror, fatalerror_notb, debug_print_traceback)
 
 
 class DebugLog(list):
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -118,3 +118,30 @@
     finally:
         debug._log = None
     assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+
+
+def test_debug_print_traceback():
+    from rpython.translator.c.test.test_genc import compile
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+
+    def ggg(n):
+        if n < 10:
+            ggg(n + 1)
+        else:
+            raise ValueError
+    def recovery():
+        llop.debug_print_traceback(lltype.Void)
+    recovery._dont_inline_ = True
+    def fff():
+        try:
+            ggg(0)
+        except:
+            recovery()
+
+    fn = compile(fff, [], return_stderr=True)
+    stderr = fn()
+    assert 'RPython traceback:\n' in stderr
+    assert stderr.count('entry_point') == 1
+    assert stderr.count('ggg') == 11
+    assert stderr.count('recovery') == 0
diff --git a/rpython/rtyper/debug.py b/rpython/rtyper/debug.py
--- a/rpython/rtyper/debug.py
+++ b/rpython/rtyper/debug.py
@@ -45,3 +45,12 @@
 fatalerror_notb._dont_inline_ = True
 fatalerror_notb._jit_look_inside_ = False
 fatalerror_notb._annenforceargs_ = [str]
+
+def debug_print_traceback():
+    # print to stderr the RPython traceback of the last caught exception,
+    # but without interrupting the program
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.debug_print_traceback(lltype.Void)
+debug_print_traceback._dont_inline_ = True
+debug_print_traceback._jit_look_inside_ = False
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -53,7 +53,8 @@
                                            unsigned_ffffffff)
 
 def compile(fn, argtypes, view=False, gcpolicy="none", backendopt=True,
-            annotatorpolicy=None, thread=False, **kwds):
+            annotatorpolicy=None, thread=False,
+            return_stderr=False, **kwds):
     argtypes_unroll = unrolling_iterable(enumerate(argtypes))
 
     for argtype in argtypes:
@@ -139,7 +140,8 @@
 
         stdout = t.driver.cbuilder.cmdexec(
             " ".join([llrepr_in(arg) for arg in args]),
-            expect_crash=(expected_exception_name is not None))
+            expect_crash=(expected_exception_name is not None),
+            err=return_stderr)
         #
         if expected_exception_name is not None:
             stdout, stderr = stdout
@@ -154,6 +156,8 @@
             assert lastline == expected or prevline == expected
             return None
 
+        if return_stderr:
+            stdout, stderr = stdout
         output(stdout)
         stdout, lastline, empty = stdout.rsplit('\n', 2)
         assert empty == ''
@@ -168,6 +172,8 @@
         else:
             assert mallocs - frees in expected_extra_mallocs
         #
+        if return_stderr:
+            return stderr
         if ll_res in [lltype.Signed, lltype.Unsigned, lltype.SignedLongLong,
                       lltype.UnsignedLongLong]:
             return int(res)

From pypy.commits at gmail.com  Mon Nov  7 10:48:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 07:48:28 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5820a24c.913fc20a.fd955.071e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88184:cce86a353c9b
Date: 2016-11-07 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/cce86a353c9b/

Log:	merge heads

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -96,7 +96,7 @@
   * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
   * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
     relevant platforms
-  * Backport the ``'faulthandler`` module from py3.5
+  * Backport the ``faulthandler`` module from py3.5
   * Improve the error message when trying to call a method where the ``self``
     parameter is missing in the definition
   * Implement ``rposix.cpu_count``

From pypy.commits at gmail.com  Mon Nov  7 11:10:17 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 07 Nov 2016 08:10:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix char result handling in _rawffi
 (argument handling still needs thinking)
Message-ID: <5820a769.e626c20a.4cc77.03bf@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88185:abdfa58326a0
Date: 2016-11-07 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/abdfa58326a0/

Log:	fix char result handling in _rawffi (argument handling still needs
	thinking)

diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py
--- a/pypy/module/_rawffi/alt/test/test_funcptr.py
+++ b/pypy/module/_rawffi/alt/test/test_funcptr.py
@@ -377,7 +377,9 @@
         libfoo = CDLL(self.libfoo_name)
         my_toupper = libfoo.getfunc('my_toupper', [types.char],
                                     types.char)
-        assert my_toupper('c') == 'C'
+        res = my_toupper(b'c')
+        assert type(res) is bytes
+        assert res == b'C'
 
     def test_unichar_args(self):
         """
diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py
--- a/pypy/module/_rawffi/alt/test/test_struct.py
+++ b/pypy/module/_rawffi/alt/test/test_struct.py
@@ -183,8 +183,8 @@
         assert struct.getfield('ulong') == sys.maxsize*2 + 1
         struct.setfield('ulong', sys.maxsize*2 + 2)
         assert struct.getfield('ulong') == 0
-        struct.setfield('char', 'a')
-        assert struct.getfield('char') == 'a'
+        struct.setfield('char', b'a')
+        assert struct.getfield('char') == b'a'
         struct.setfield('unichar', '\u1234')
         assert struct.getfield('unichar') == '\u1234'
         struct.setfield('ptr', -1)
diff --git a/pypy/module/_rawffi/alt/type_converter.py b/pypy/module/_rawffi/alt/type_converter.py
--- a/pypy/module/_rawffi/alt/type_converter.py
+++ b/pypy/module/_rawffi/alt/type_converter.py
@@ -225,7 +225,7 @@
             return space.wrap(uintval)
         elif w_ffitype.is_char():
             ucharval = self.get_char(w_ffitype)
-            return space.wrap(chr(ucharval))
+            return space.newbytes(chr(ucharval))
         elif w_ffitype.is_unichar():
             wcharval = self.get_unichar(w_ffitype)
             return space.wrap(unichr(wcharval))

From pypy.commits at gmail.com  Mon Nov  7 11:50:41 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 08:50:41 -0800 (PST)
Subject: [pypy-commit] pypy default: attempt to fix 12160ae9198a
Message-ID: <5820b0e1.958c1c0a.1fb40.6de7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88186:c85d4dd82b49
Date: 2016-11-07 18:44 +0200
http://bitbucket.org/pypy/pypy/changeset/c85d4dd82b49/

Log:	attempt to fix 12160ae9198a

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -34,19 +34,37 @@
                     " or contribute the missing support in PyPy.")
     return _get_compiler_type(cc, True)
 
+def _find_vcvarsall(version):
+    # copied from setuptools.msvc9_support.py
+    from distutils.msvc9compiler import Reg
+    VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f'
+    key = VC_BASE % ('', version)
+    try:
+        # Per-user installs register the compiler path here
+        productdir = Reg.get_value(key, "installdir")
+    except KeyError:
+        try:
+            # All-user installs on a 64-bit system register here
+            key = VC_BASE % ('Wow6432Node\\', version)
+            productdir = Reg.get_value(key, "installdir")
+        except KeyError:
+            productdir = None
+
+    if productdir:
+        vcvarsall = os.path.join(productdir, "vcvarsall.bat")
+        if os.path.isfile(vcvarsall):
+            return vcvarsall
+    return None
+
 def _get_msvc_env(vsver, x64flag):
     vcvars = None
     try:
         toolsdir = os.environ['VS%sCOMNTOOLS' % vsver]
     except KeyError:
-        # try to import from the "Microsoft Visual C++ Compiler for Python 2.7"
-        try:
-            import setuptools
-        except ImportError:
-            return None
+        # try to import from the registry, as done in setuptools
         # XXX works for 90 but is it generalizable?
         toolsdir = ''
-        vcvars = setuptools.msvc.msvc9_find_vcvarsall(vsver/10)
+        vcvars = _find_vcvarsall(vsver/10)
 
     if not vcvars:
         if x64flag:

From pypy.commits at gmail.com  Mon Nov  7 11:50:43 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 08:50:43 -0800 (PST)
Subject: [pypy-commit] pypy default: always turn on /LARGEADDRESSAWARE
Message-ID: <5820b0e3.958c1c0a.1fb40.6deb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88187:4a802afbf7de
Date: 2016-11-07 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/4a802afbf7de/

Log:	always turn on /LARGEADDRESSAWARE

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -130,7 +130,7 @@
     link = 'link.exe'
 
     cflags = ('/MD', '/O2', '/Zi')
-    link_flags = ('/debug',)
+    link_flags = ('/debug','/LARGEADDRESSAWARE')
     standalone_only = ()
     shared_only = ()
     environ = None

From pypy.commits at gmail.com  Mon Nov  7 12:22:29 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 09:22:29 -0800 (PST)
Subject: [pypy-commit] pypy default: close file once,
 second close will segfault win32
Message-ID: <5820b855.8b901c0a.4e4a5.81d1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88188:57ee4821e42c
Date: 2016-11-07 19:20 +0200
http://bitbucket.org/pypy/pypy/changeset/57ee4821e42c/

Log:	close file once, second close will segfault win32

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -140,9 +140,8 @@
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
-        os.close(c_fileno(fp))
+        c_fclose(fp)
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 

From pypy.commits at gmail.com  Mon Nov  7 12:31:40 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 09:31:40 -0800 (PST)
Subject: [pypy-commit] pypy default: fix 57ee4821e42c differently
Message-ID: <5820ba7c.43921c0a.73dc2.7f3e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88189:fe2af558d879
Date: 2016-11-07 19:28 +0200
http://bitbucket.org/pypy/pypy/changeset/fe2af558d879/

Log:	fix 57ee4821e42c differently

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,8 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
+from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
+                Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -140,8 +141,10 @@
 
         # try again, but with a closed file
         fp = c_fopen(str(filepath), "rb")
-        c_fclose(fp)
+        os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
+        if is_valid_fd(c_fileno(fp)):
+            c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 

From pypy.commits at gmail.com  Mon Nov  7 13:01:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 10:01:36 -0800 (PST)
Subject: [pypy-commit] pypy default: monkeypatch distutils for the
 "microsoft visual c++ for python" suite via setuptools
Message-ID: <5820c180.a115c20a.26a18.3b37@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88190:05a2db282b48
Date: 2016-11-07 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/05a2db282b48/

Log:	monkeypatch distutils for the "microsoft visual c++ for python"
	suite via setuptools

diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -192,6 +192,13 @@
 
 def _build(cfilenames, outputfilename, compile_extra, link_extra,
         include_dirs, libraries, library_dirs):
+    try:
+        # monkeypatch distutils for some versions of msvc compiler
+        import setuptools
+    except ImportError:
+        # XXX if this fails and is required, 
+        #     we must call pypy -mensurepip after translation
+        pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
     compiler = new_compiler(force=1)

From pypy.commits at gmail.com  Mon Nov  7 13:05:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 10:05:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use the same logic as default for the
 Return and Yield exceptions.
Message-ID: <5820c258.6252c20a.214f3.456c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88191:b9832c7d5255
Date: 2016-11-07 19:04 +0100
http://bitbucket.org/pypy/pypy/changeset/b9832c7d5255/

Log:	Use the same logic as default for the Return and Yield exceptions.
	Should fix the major issue with the JIT performance right now.

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -307,19 +307,18 @@
             # last_instr is -1.  After a generator suspends it points to
             # the YIELD_VALUE/YIELD_FROM instruction.
             try:
-                if in_generator is None:
-                    assert self.last_instr == -1
-                    next_instr = r_uint(0)
+                try:
+                    if in_generator is None:
+                        assert self.last_instr == -1
+                        next_instr = r_uint(0)
+                    else:
+                        next_instr = in_generator.resume_execute_frame(
+                                                        self, w_arg_or_err)
+                except pyopcode.Yield:
+                    w_exitvalue = self.popvalue()
                 else:
-                    next_instr = in_generator.resume_execute_frame(
-                                                    self, w_arg_or_err)
-                #
-                self.dispatch(self.pycode, next_instr, executioncontext)
-            except pyopcode.Return:
-                self.last_exception = None
-                w_exitvalue = self.popvalue()
-            except pyopcode.Yield:
-                w_exitvalue = self.popvalue()
+                    w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                                executioncontext)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -60,8 +60,14 @@
         # For the sequel, force 'next_instr' to be unsigned for performance
         next_instr = r_uint(next_instr)
         co_code = pycode.co_code
-        while True:
-            next_instr = self.handle_bytecode(co_code, next_instr, ec)
+        try:
+            while True:
+                next_instr = self.handle_bytecode(co_code, next_instr, ec)
+        except Return:
+            self.last_exception = None
+            return self.popvalue()
+        except Yield:
+            return self.popvalue()
 
     def handle_bytecode(self, co_code, next_instr, ec):
         try:
@@ -1539,6 +1545,7 @@
 
 ### ____________________________________________________________ ###
 
+
 class Return(Exception):
     """Raised when exiting a frame via a 'return' statement."""
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -91,8 +91,12 @@
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
                 is_being_profiled = self.get_is_being_profiled()
         except Yield:
+            w_result = self.popvalue()
             jit.hint(self, force_virtualizable=True)
-            raise
+            return w_result
+        except Return:
+            self.last_exception = None
+            return self.popvalue()
 
     def jump_absolute(self, jumpto, ec):
         if we_are_jitted():

From pypy.commits at gmail.com  Mon Nov  7 13:09:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 07 Nov 2016 10:09:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: oups
Message-ID: <5820c368.0a22c20a.122a9.4185@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88192:6d9711ba03d9
Date: 2016-11-07 19:09 +0100
http://bitbucket.org/pypy/pypy/changeset/6d9711ba03d9/

Log:	oups

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -15,7 +15,7 @@
 from pypy.interpreter.pycode import CO_GENERATOR, CO_COROUTINE, PyCode
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pyopcode import Yield
+from pypy.interpreter.pyopcode import Yield, Return
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app

From pypy.commits at gmail.com  Mon Nov  7 13:26:44 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 07 Nov 2016 10:26:44 -0800 (PST)
Subject: [pypy-commit] pypy default: introduce @not_rpython decorator
 instead of the weird "let's put something into
Message-ID: <5820c764.8b9a1c0a.9f7d5.6a29@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88193:24605a73710b
Date: 2016-11-07 18:17 +0100
http://bitbucket.org/pypy/pypy/changeset/24605a73710b/

Log:	introduce @not_rpython decorator instead of the weird "let's put
	something into the docstring" approach

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -146,6 +146,27 @@
         # result should be an integer
         assert s.knowntype == int
 
+    def test_not_rpython(self):
+        def g(x):
+            """ NOT_RPYTHON """
+            return eval(x)
+        def f(x):
+            return g(str(x))
+        a = self.RPythonAnnotator()
+        with py.test.raises(ValueError):
+            a.build_types(f, [int])
+
+    def test_not_rpython_decorator(self):
+        from rpython.rlib.objectmodel import not_rpython
+        @not_rpython
+        def g(x):
+            return eval(x)
+        def f(x):
+            return g(str(x))
+        a = self.RPythonAnnotator()
+        with py.test.raises(ValueError):
+            a.build_types(f, [int])
+
     def test_lists(self):
         a = self.RPythonAnnotator()
         end_cell = a.build_types(snippet.poor_man_rev_range, [int])
diff --git a/rpython/doc/faq.rst b/rpython/doc/faq.rst
--- a/rpython/doc/faq.rst
+++ b/rpython/doc/faq.rst
@@ -80,6 +80,9 @@
 stops and reports this as an error. You can therefore mark functions as
 "NOT_RPYTHON" to make sure that they are never analyzed.
 
+This method of marking a function as not RPython is outdated. For new code,
+please use the ``@objectmodel.not_rpython`` decorator instead.
+
 
 Couldn't we simply take a Python syntax tree and turn it into Lisp?
 -------------------------------------------------------------------
@@ -143,7 +146,7 @@
   attempt to point newcomers at existing alternatives, which are more
   mainstream and where they will get help from many people.*
 
-  *If anybody seriously wants to promote RPython anyway, he is welcome
+  *If anybody seriously wants to promote RPython anyway, they are welcome
   to: we won't actively resist such a plan.  There are a lot of things
   that could be done to make RPython a better Java-ish language for
   example, starting with supporting non-GIL-based multithreading, but we
@@ -182,8 +185,8 @@
 patch the generated machine code.
 
 So the position of the core PyPy developers is that if anyone wants to
-make an N+1'th attempt with LLVM, he is welcome, and he will receive a
-bit of help on the IRC channel, but he is left with the burden of proof
+make an N+1'th attempt with LLVM, they are welcome, and they will receive a
+bit of help on the IRC channel, but they are left with the burden of proof
 that it works.
 
 
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -18,6 +18,8 @@
     except AttributeError:
         raise ValueError("%r is not RPython: it is likely an unexpected "
                          "built-in function or type" % (func,))
+    if getattr(func, "_not_rpython_", False):
+        raise ValueError("%r is tagged as @not_rpython" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -223,6 +223,13 @@
     func._always_inline_ = 'try'
     return func
 
+def not_rpython(func):
+    """ mark a function as not rpython. the translation process will raise an
+    error if it encounters the function. """
+    # test is in annotator/test/test_annrpython.py
+    func._not_rpython_ = True
+    return func
+
 
 # ____________________________________________________________
 
@@ -545,8 +552,9 @@
     return intmask(x)
 TAKE_NEXT = float(2**31)
 
+ at not_rpython
 def _hash_tuple(t):
-    """NOT_RPYTHON.  The algorithm behind compute_hash() for a tuple.
+    """The algorithm behind compute_hash() for a tuple.
     It is modelled after the old algorithm of Python 2.3, which is
     a bit faster than the one introduced by Python 2.4.  We assume
     that nested tuples are very uncommon in RPython, making the bad

From pypy.commits at gmail.com  Mon Nov  7 13:26:46 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 07 Nov 2016 10:26:46 -0800 (PST)
Subject: [pypy-commit] pypy default: replace NOT_RPYTHON by @not_rpython in
 rlib/
Message-ID: <5820c766.94841c0a.a2a12.99d1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88194:959d4c580113
Date: 2016-11-07 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/959d4c580113/

Log:	replace NOT_RPYTHON by @not_rpython in rlib/

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -3,7 +3,7 @@
 import py
 
 from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.sourcetools import rpython_wrapper
@@ -614,14 +614,14 @@
     inline_jit_merge_point = False
     _store_last_enter_jit = None
 
+    @not_rpython
     def __init__(self, greens=None, reds=None, virtualizables=None,
                  get_jitcell_at=None, set_jitcell_at=None,
                  get_printable_location=None, confirm_enter_jit=None,
                  can_never_inline=None, should_unroll_one_iteration=None,
                  name='jitdriver', check_untranslated=True, vectorize=False,
                  get_unique_id=None, is_recursive=False, get_location=None):
-        """ NOT_RPYTHON
-            get_location:
+        """get_location:
               The return value is designed to provide enough information to express the
               state of an interpreter when invoking jit_merge_point.
               For a bytecode interperter such as PyPy this includes, filename, line number,
diff --git a/rpython/rlib/parsing/parsing.py b/rpython/rlib/parsing/parsing.py
--- a/rpython/rlib/parsing/parsing.py
+++ b/rpython/rlib/parsing/parsing.py
@@ -1,6 +1,7 @@
 import py
 from rpython.rlib.parsing.lexer import SourcePos
 from rpython.rlib.parsing.tree import Node, Symbol, Nonterminal
+from rpython.rlib.objectmodel import not_rpython
 
 class Rule(object):
     def __init__(self, nonterminal, expansions):
@@ -181,8 +182,8 @@
             raise ParseError(input[error.pos].source_pos, error)
         return result[0]
 
+    @not_rpython
     def has_left_recursion(self):
-        """NOT_RPYTHON"""
         follows = {}
         for rule in self.rules:
             follow = py.builtin.set()
diff --git a/rpython/rlib/parsing/tree.py b/rpython/rlib/parsing/tree.py
--- a/rpython/rlib/parsing/tree.py
+++ b/rpython/rlib/parsing/tree.py
@@ -1,4 +1,5 @@
 import py
+from rpython.rlib.objectmodel import not_rpython
 
 class Node(object):
     def view(self):
@@ -38,8 +39,8 @@
             id(self), symbol,
             repr(addinfo).replace('"', '').replace("\\", "\\\\")))
 
+    @not_rpython
     def visit(self, visitor):
-        "NOT_RPYTHON"
         if isinstance(visitor, RPythonVisitor):
             return visitor.dispatch(self)
         method = getattr(visitor, "visit_" + self.symbol, None)
@@ -76,8 +77,8 @@
                     id(child),
                     repr(child).replace('"', '').replace("\\", "\\\\"))
 
+    @not_rpython
     def visit(self, visitor):
-        "NOT_RPYTHON"
         if isinstance(visitor, RPythonVisitor):
             return visitor.dispatch(self)
         general = getattr(visitor, "visit", None)
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -37,7 +37,7 @@
 from rpython.rlib import objectmodel
 from rpython.flowspace.model import Constant, const
 from rpython.flowspace.specialcase import register_flow_sc
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, not_rpython
 
 """
 Long-term target:
@@ -105,10 +105,8 @@
 # XXX returning int(n) should not be necessary and should be simply n.
 # XXX TODO: replace all int(n) by long(n) and fix everything that breaks.
 # XXX       Then relax it and replace int(n) by n.
+ at not_rpython
 def intmask(n):
-    """
-    NOT_RPYTHON
-    """
     if isinstance(n, objectmodel.Symbolic):
         return n        # assume Symbolics don't overflow
     assert not isinstance(n, float)
@@ -120,10 +118,8 @@
         n -= 2*LONG_TEST
     return int(n)
 
+ at not_rpython
 def longlongmask(n):
-    """
-    NOT_RPYTHON
-    """
     assert isinstance(n, (int, long))
     n = long(n)
     n &= LONGLONG_MASK
@@ -168,8 +164,8 @@
     return isinstance(r, (base_int, int, long, bool)) and (
         -maxint - 1 <= r <= maxint)
 
+ at not_rpython
 def ovfcheck(r):
-    "NOT_RPYTHON"
     # to be used as ovfcheck(x <op> y)
     # raise OverflowError if the operation did overflow
     assert not isinstance(r, r_uint), "unexpected ovf check on unsigned"
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -6,7 +6,7 @@
 #
 import sys, weakref
 from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rlib import rgc
 
@@ -23,8 +23,9 @@
     return res
 
 
+ at not_rpython
 def init(dealloc_trigger_callback=None):
-    """NOT_RPYTHON: set up rawrefcount with the GC.  This is only used
+    """set up rawrefcount with the GC.  This is only used
     for tests; it should not be called at all during translation.
     """
     global _p_list, _o_list, _adr2pypy, _pypy2ob, _pypy2ob_rev
@@ -37,8 +38,9 @@
     _d_list = []
     _dealloc_trigger_callback = dealloc_trigger_callback
 
+ at not_rpython
 def create_link_pypy(p, ob):
-    "NOT_RPYTHON: a link where the PyPy object contains some or all the data"
+    "a link where the PyPy object contains some or all the data"
     #print 'create_link_pypy\n\t%s\n\t%s' % (p, ob)
     assert p not in _pypy2ob
     assert ob._obj not in _pypy2ob_rev
@@ -48,8 +50,9 @@
     _pypy2ob_rev[ob._obj] = p
     _p_list.append(ob)
 
+ at not_rpython
 def create_link_pyobj(p, ob):
-    """NOT_RPYTHON: a link where the PyObject contains all the data.
+    """a link where the PyObject contains all the data.
        from_obj() will not work on this 'p'."""
     #print 'create_link_pyobj\n\t%s\n\t%s' % (p, ob)
     assert p not in _pypy2ob
@@ -58,8 +61,8 @@
     ob.c_ob_pypy_link = _build_pypy_link(p)
     _o_list.append(ob)
 
+ at not_rpython
 def from_obj(OB_PTR_TYPE, p):
-    "NOT_RPYTHON"
     ob = _pypy2ob.get(p)
     if ob is None:
         return lltype.nullptr(OB_PTR_TYPE.TO)
@@ -67,8 +70,8 @@
     assert _pypy2ob_rev[ob._obj] is p
     return ob
 
+ at not_rpython
 def to_obj(Class, ob):
-    "NOT_RPYTHON"
     link = ob.c_ob_pypy_link
     if link == 0:
         return None
@@ -76,8 +79,9 @@
     assert isinstance(p, Class)
     return p
 
+ at not_rpython
 def next_dead(OB_PTR_TYPE):
-    """NOT_RPYTHON.  When the GC runs, it finds some pyobjs to be dead
+    """When the GC runs, it finds some pyobjs to be dead
     but cannot immediately dispose of them (it doesn't know how to call
     e.g. tp_dealloc(), and anyway calling it immediately would cause all
     sorts of bugs).  So instead, it stores them in an internal list,
@@ -89,8 +93,9 @@
     assert lltype.typeOf(ob) == OB_PTR_TYPE
     return ob
 
+ at not_rpython
 def _collect(track_allocation=True):
-    """NOT_RPYTHON: for tests only.  Emulates a GC collection.
+    """for tests only.  Emulates a GC collection.
     Will invoke dealloc_trigger_callback() once if there are objects
     whose _Py_Dealloc() should be called.
     """
diff --git a/rpython/rlib/rawstorage.py b/rpython/rlib/rawstorage.py
--- a/rpython/rlib/rawstorage.py
+++ b/rpython/rlib/rawstorage.py
@@ -4,7 +4,7 @@
 from rpython.annotator import model as annmodel
 from rpython.rtyper.llannotation import lltype_to_annotation
 from rpython.rlib.rgc import lltype_is_gc
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, not_rpython
 
 RAW_STORAGE = rffi.CCHARP.TO
 RAW_STORAGE_PTR = rffi.CCHARP
@@ -16,23 +16,23 @@
                          track_allocation=track_allocation,
                          zero=zero)
 
+ at not_rpython
 def raw_storage_getitem(TP, storage, index):
-    "NOT_RPYTHON"
     _check_alignment(TP, index)
     return _raw_storage_getitem_unchecked(TP, storage, index)
 
+ at not_rpython
 def _raw_storage_getitem_unchecked(TP, storage, index):
-    "NOT_RPYTHON"
     return rffi.cast(rffi.CArrayPtr(TP), rffi.ptradd(storage, index))[0]
 
+ at not_rpython
 def raw_storage_setitem(storage, index, item):
-    "NOT_RPYTHON"
     TP = lltype.typeOf(item)
     _check_alignment(TP, index)
     _raw_storage_setitem_unchecked(storage, index, item)
 
+ at not_rpython
 def _raw_storage_setitem_unchecked(storage, index, item):
-    "NOT_RPYTHON"
     TP = lltype.typeOf(item)
     rffi.cast(rffi.CArrayPtr(TP), rffi.ptradd(storage, index))[0] = item
 
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -4,7 +4,7 @@
 from rpython.rlib.rfloat import isinf, isnan
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.debug import make_sure_not_resized, check_regular_int
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, not_rpython
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper import extregistry
@@ -149,14 +149,14 @@
         self.sign = sign
 
     # __eq__ and __ne__ method exist for testingl only, they are not RPython!
+    @not_rpython
     def __eq__(self, other):
-        # NOT_RPYTHON
         if not isinstance(other, rbigint):
             return NotImplemented
         return self.eq(other)
 
+    @not_rpython
     def __ne__(self, other):
-        # NOT_RPYTHON
         return not (self == other)
 
     def digit(self, x):
@@ -218,8 +218,8 @@
         return NULLRBIGINT
 
     @staticmethod
+    @not_rpython
     def fromlong(l):
-        "NOT_RPYTHON"
         return rbigint(*args_from_long(l))
 
     @staticmethod
@@ -1230,8 +1230,8 @@
             ret /= math.log(base)
         return ret
 
+    @not_rpython
     def tolong(self):
-        "NOT_RPYTHON"
         l = 0L
         digits = list(self._digits)
         digits.reverse()
@@ -1356,8 +1356,8 @@
 # ^^^ specialized by the precise type of 'x', which is typically a r_xxx
 #     instance from rlib.rarithmetic
 
+ at not_rpython
 def args_from_long(x):
-    "NOT_RPYTHON"
     if x >= 0:
         if x == 0:
             return [NULLDIGIT], 0
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -3,7 +3,7 @@
 
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.rarithmetic import r_uint
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
@@ -92,8 +92,9 @@
             return ""
         return rffi.charp2str(res)
 
+    @not_rpython
     def _dlerror_on_dlopen_untranslated(name):
-        "NOT_RPYTHON: aaargh"
+        # aaargh
         import ctypes
         name = rffi.charp2str(name)
         try:
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -7,6 +7,7 @@
 from rpython.rtyper.extfunc import register_external
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.objectmodel import not_rpython
 
 
 class CConfig:
@@ -187,16 +188,17 @@
     # Try to get math functions added in 2.6.
     from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p
 except ImportError:
+    @not_rpython
     def isinf(x):
-        "NOT_RPYTHON"
         return x == INFINITY or x == -INFINITY
 
+    @not_rpython
     def isnan(v):
-        "NOT_RPYTHON"
         return v != v
 
+    @not_rpython
     def copysign(x, y):
-        """NOT_RPYTHON. Return x with the sign of y"""
+        """Return x with the sign of y"""
         if x < 0.:
             x = -x
         if y > 0. or (y == 0. and math.atan2(y, -1.) > 0.):
@@ -208,8 +210,8 @@
     _2_to_p28 = 268435456.0; # 2**28
     _ln2 = 6.93147180559945286227E-01
 
+    @not_rpython
     def acosh(x):
-        "NOT_RPYTHON"
         if isnan(x):
             return NAN
         if x < 1.:
@@ -227,8 +229,8 @@
         t = x - 1.0
         return log1p(t + math.sqrt(2. * t + t * t))
 
+    @not_rpython
     def asinh(x):
-        "NOT_RPYTHON"
         absx = abs(x)
         if not isfinite(x):
             return x
@@ -243,8 +245,8 @@
             w = log1p(absx + t / (1. + math.sqrt(1. + t)))
         return copysign(w, x)
 
+    @not_rpython
     def atanh(x):
-        "NOT_RPYTHON"
         if isnan(x):
             return x
         absx = abs(x)
@@ -259,8 +261,8 @@
             t = .5 * log1p((absx + absx) / (1. - absx))
         return copysign(t, x)
 
+    @not_rpython
     def log1p(x):
-        "NOT_RPYTHON"
         if abs(x) < DBL_EPSILON // 2.:
             return x
         elif -.5 <= x <= 1.:
@@ -272,8 +274,8 @@
 try:
     from math import expm1 # Added in Python 2.7.
 except ImportError:
+    @not_rpython
     def expm1(x):
-        "NOT_RPYTHON"
         if abs(x) < .7:
             u = math.exp(x)
             if u == 1.:
@@ -319,8 +321,8 @@
         r = math.floor(absx)
     return copysign(r, x)
 
+ at not_rpython
 def isfinite(x):
-    "NOT_RPYTHON"
     return not isinf(x) and not isnan(x)
 
 def float_as_rbigint_ratio(value):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -5,7 +5,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, enforceargs, specialize
-from rpython.rlib.objectmodel import CDefinedIntSymbolic
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, not_rpython
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.lltypesystem import lltype, llmemory
 
@@ -422,8 +422,9 @@
         else:
             self._untranslated_register_finalizer(obj)
 
+    @not_rpython
     def _get_tag(self):
-        "NOT_RPYTHON: special-cased below"
+        "special-cased below"
 
     def _reset(self):
         import collections
@@ -530,14 +531,14 @@
 # ____________________________________________________________
 
 
+ at not_rpython
 def get_rpy_roots():
-    "NOT_RPYTHON"
     # Return the 'roots' from the GC.
     # The gc typically returns a list that ends with a few NULL_GCREFs.
     return [_GcRef(x) for x in gc.get_objects()]
 
+ at not_rpython
 def get_rpy_referents(gcref):
-    "NOT_RPYTHON"
     x = gcref._x
     if isinstance(x, list):
         d = x
@@ -583,8 +584,8 @@
                          resulttype=lltype.Void)
 
 
+ at not_rpython
 def get_rpy_memory_usage(gcref):
-    "NOT_RPYTHON"
     # approximate implementation using CPython's type info
     Class = type(gcref._x)
     size = Class.__basicsize__
@@ -592,8 +593,8 @@
         size += Class.__itemsize__ * len(gcref._x)
     return size
 
+ at not_rpython
 def get_rpy_type_index(gcref):
-    "NOT_RPYTHON"
     from rpython.rlib.rarithmetic import intmask
     Class = gcref._x.__class__
     return intmask(id(Class))
@@ -607,33 +608,33 @@
     else:
         return id(gcref._x)
 
+ at not_rpython
 def dump_rpy_heap(fd):
-    "NOT_RPYTHON"
     raise NotImplementedError
 
+ at not_rpython
 def get_typeids_z():
-    "NOT_RPYTHON"
     raise NotImplementedError
 
+ at not_rpython
 def get_typeids_list():
-    "NOT_RPYTHON"
     raise NotImplementedError
 
+ at not_rpython
 def has_gcflag_extra():
-    "NOT_RPYTHON"
     return True
 has_gcflag_extra._subopnum = 1
 
 _gcflag_extras = set()
 
+ at not_rpython
 def get_gcflag_extra(gcref):
-    "NOT_RPYTHON"
     assert gcref   # not NULL!
     return gcref in _gcflag_extras
 get_gcflag_extra._subopnum = 2
 
+ at not_rpython
 def toggle_gcflag_extra(gcref):
-    "NOT_RPYTHON"
     assert gcref   # not NULL!
     try:
         _gcflag_extras.remove(gcref)
@@ -801,12 +802,12 @@
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
+ at not_rpython
 def _is_rpy_instance(gcref):
-    "NOT_RPYTHON"
     raise NotImplementedError
 
+ at not_rpython
 def _get_llcls_from_cls(Class):
-    "NOT_RPYTHON"
     raise NotImplementedError
 
 class Entry(ExtRegistryEntry):
diff --git a/rpython/rlib/rgil.py b/rpython/rlib/rgil.py
--- a/rpython/rlib/rgil.py
+++ b/rpython/rlib/rgil.py
@@ -3,6 +3,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.extregistry import ExtRegistryEntry
+from rpython.rlib.objectmodel import not_rpython
 
 # these functions manipulate directly the GIL, whose definition does not
 # escape the C code itself
@@ -58,8 +59,8 @@
 
 _emulated_after_thread_switch = None
 
+ at not_rpython
 def _after_thread_switch():
-    """NOT_RPYTHON"""
     if _emulated_after_thread_switch is not None:
         _emulated_after_thread_switch()
 
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.error import TyperError
 
 from rpython.rlib._os_support import _preferred_traits, string_traits
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, not_rpython
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
@@ -240,9 +240,9 @@
 
 s_StatResult = SomeStatResult()
 
-
+ at not_rpython
 def make_stat_result(tup):
-    """NOT_RPYTHON: Turn a tuple into an os.stat_result object."""
+    """Turn a tuple into an os.stat_result object."""
     assert len(tup) == len(STAT_FIELDS)
     assert float not in [type(x) for x in tup]
     positional = []
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -3,7 +3,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.rsre import rsre_char
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib import jit
 from rpython.rlib.rsre.rsre_jit import install_jitdriver, install_jitdriver_spec
 
@@ -121,16 +121,18 @@
         assert result >= 0
         return result
 
+    @not_rpython
     def str(self, index):
-        """NOT_RPYTHON: Must be overridden in a concrete subclass.
+        """Must be overridden in a concrete subclass.
         The tag ^^^ here is used to generate a translation-time crash
         if there is a call to str() that is indirect.  All calls must
         be direct for performance reasons; you need to specialize the
         caller with @specializectx."""
         raise NotImplementedError
 
+    @not_rpython
     def lowstr(self, index):
-        """NOT_RPYTHON: Similar to str()."""
+        """Similar to str()."""
         raise NotImplementedError
 
     def get_mark(self, gid):
diff --git a/rpython/rlib/rsre/rsre_re.py b/rpython/rlib/rsre/rsre_re.py
--- a/rpython/rlib/rsre/rsre_re.py
+++ b/rpython/rlib/rsre/rsre_re.py
@@ -2,7 +2,7 @@
 This is not used in a PyPy translation, but it can be used
 in RPython code.  It exports the same interface as the
 Python 're' module.  You can call the functions at the start
-of the module (expect the ones with NOT_RPYTHON for now).
+of the module (expect the ones with @not_rpython for now).
 They must be called with a *constant* pattern string.
 """
 import re, sys
@@ -10,6 +10,7 @@
 from rpython.rlib.rsre.rpy import get_code as _get_code
 from rpython.rlib.unicodedata import unicodedb
 from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import not_rpython
 rsre_char.set_unicode_db(unicodedb)
 
 
@@ -37,12 +38,12 @@
 def finditer(pattern, string, flags=0):
     return compile(pattern, flags).finditer(string)
 
+ at not_rpython
 def sub(pattern, repl, string, count=0):
-    "NOT_RPYTHON"
     return compile(pattern).sub(repl, string, count)
 
+ at not_rpython
 def subn(pattern, repl, string, count=0):
-    "NOT_RPYTHON"
     return compile(pattern).subn(repl, string, count)
 
 @specialize.call_location()
@@ -103,8 +104,8 @@
                 break
             yield match
 
+    @not_rpython
     def subn(self, repl, string, count=0):
-        "NOT_RPYTHON"
         filter = repl
         if not callable(repl) and "\\" in repl:
             # handle non-literal strings; hand it over to the template compiler
@@ -151,8 +152,8 @@
             item = "".join(sublist)
         return item, n
 
+    @not_rpython
     def sub(self, repl, string, count=0):
-        "NOT_RPYTHON"
         item, n = self.subn(repl, string, count)
         return item
 
diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py
--- a/rpython/rlib/rstrategies/rstrategies.py
+++ b/rpython/rlib/rstrategies/rstrategies.py
@@ -2,7 +2,7 @@
 import weakref, sys
 from rpython.rlib.rstrategies import logger
 from rpython.rlib import jit, objectmodel, rerased
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, not_rpython
 
 def make_accessors(strategy='strategy', storage='storage'):
     """
@@ -155,6 +155,7 @@
                 return strategy_type
         raise ValueError("Could not find strategy to handle: %s" % objects)
 
+    @not_rpython
     def decorate_strategies(self, transitions):
         """
         As an alternative to decorating all strategies with @strategy,
@@ -162,7 +163,6 @@
         calling __init__. transitions is a dict mapping all strategy classes to
         their 'generalize' list parameter (see @strategy decorator).
         """
-        "NOT_RPYTHON"
         for strategy_class, generalized in transitions.items():
             strategy(generalized)(strategy_class)
 
@@ -217,8 +217,8 @@
     # Internal methods
     # =============================
 
+    @not_rpython
     def _patch_strategy_class(self, strategy_class, root_class):
-        "NOT_RPYTHON"
         # Patch root class: Add default handler for visitor
         def _convert_storage_from_OTHER(self, w_self, previous_strategy):
             self._convert_storage_from(w_self, previous_strategy)
@@ -231,16 +231,16 @@
             getattr(new_strategy, funcname)(w_self, self)
         strategy_class._convert_storage_to = _convert_storage_to
 
+    @not_rpython
     def _collect_subclasses(self, cls):
-        "NOT_RPYTHON"
         subclasses = []
         for subcls in cls.__subclasses__():
             subclasses.append(subcls)
             subclasses.extend(self._collect_subclasses(subcls))
         return subclasses
 
+    @not_rpython
     def _order_strategies(self):
-        "NOT_RPYTHON"
         def get_generalization_depth(strategy, visited=None):
             if visited is None:
                 visited = set()
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -6,7 +6,7 @@
     SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePBC)
 from rpython.rtyper.llannotation import SomePtr
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import newlist_hint, resizelist_hint, specialize
+from rpython.rlib.objectmodel import newlist_hint, resizelist_hint, specialize, not_rpython
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT as BLOOM_WIDTH
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb
@@ -516,37 +516,37 @@
 class AbstractStringBuilder(object):
     # This is not the real implementation!
 
+    @not_rpython
     def __init__(self, init_size=INIT_SIZE):
-        "NOT_RPYTHON"
         self._l = []
         self._size = 0
 
+    @not_rpython
     def _grow(self, size):
-        "NOT_RPYTHON"
         self._size += size
 
+    @not_rpython
     def append(self, s):
-        "NOT_RPYTHON"
         assert isinstance(s, self._tp)
         self._l.append(s)
         self._grow(len(s))
 
+    @not_rpython
     def append_slice(self, s, start, end):
-        "NOT_RPYTHON"
         assert isinstance(s, self._tp)
         assert 0 <= start <= end <= len(s)
         s = s[start:end]
         self._l.append(s)
         self._grow(len(s))
 
+    @not_rpython
     def append_multiple_char(self, c, times):
-        "NOT_RPYTHON"
         assert isinstance(c, self._tp)
         self._l.append(c * times)
         self._grow(times)
 
+    @not_rpython
     def append_charpsize(self, s, size):
-        "NOT_RPYTHON"
         assert size >= 0
         l = []
         for i in xrange(size):
@@ -554,15 +554,15 @@
         self._l.append(self._tp("").join(l))
         self._grow(size)
 
+    @not_rpython
     def build(self):
-        "NOT_RPYTHON"
         result = self._tp("").join(self._l)
         assert len(result) == self._size
         self._l = [result]
         return result
 
+    @not_rpython
     def getlength(self):
-        "NOT_RPYTHON"
         return self._size
 
 
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -5,7 +5,7 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.debug import ll_assert
 from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib.objectmodel import CDefinedIntSymbolic
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, not_rpython
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -37,8 +37,8 @@
     return rffi.llexternal(name, args, result, compilation_info=eci,
                            **kwds)
 
+ at not_rpython
 def _emulated_start_new_thread(func):
-    "NOT_RPYTHON"
     import thread
     try:
         ident = thread.start_new_thread(func, ())
@@ -319,8 +319,9 @@
 
 
 class ThreadLocalField(object):
+    @not_rpython
     def __init__(self, FIELDTYPE, fieldname, loop_invariant=False):
-        "NOT_RPYTHON: must be prebuilt"
+        "must be prebuilt"
         try:
             from thread import _local
         except ImportError:
@@ -328,7 +329,7 @@
                 pass
         self.FIELDTYPE = FIELDTYPE
         self.fieldname = fieldname
-        self.local = _local()      # <- NOT_RPYTHON
+        self.local = _local()      # <- not rpython
         zero = rffi.cast(FIELDTYPE, 0)
         offset = CDefinedIntSymbolic('RPY_TLOFS_%s' % self.fieldname,
                                      default='?')
@@ -381,8 +382,9 @@
     # leak the objects when a thread finishes; see threadlocal.c.)
     _COUNT = 1
 
+    @not_rpython
     def __init__(self, Cls, loop_invariant=False):
-        "NOT_RPYTHON: must be prebuilt"
+        "must be prebuilt"
         self.Cls = Cls
         unique_id = ThreadLocalReference._COUNT
         ThreadLocalReference._COUNT += 1
@@ -454,8 +456,9 @@
     tlfield_rpy_lasterror = ThreadLocalField(rwin32.DWORD, "rpy_lasterror")
     tlfield_alt_lasterror = ThreadLocalField(rwin32.DWORD, "alt_lasterror")
 
+ at not_rpython
 def _threadlocalref_seeme(field):
-    "NOT_RPYTHON"
+    pass
 
 class _Entry(ExtRegistryEntry):
     _about_ = _threadlocalref_seeme
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -6,6 +6,7 @@
 import errno
 
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.objectmodel import not_rpython
 
 
 if sys.platform == 'win32':
@@ -46,8 +47,9 @@
         compilation_info=eci,
         save_err=rffi.RFFI_SAVE_LASTERROR)
 
+    @not_rpython
     def init_urandom():
-        """NOT_RPYTHON
+        """
         Return an array of one HCRYPTPROV, initialized to NULL.
         It is filled automatically the first time urandom() is called.
         """
@@ -84,9 +86,8 @@
                 raise ValueError("RAND_pseudo_bytes")
             return buf.str(n)
 else:  # Posix implementation
+    @not_rpython
     def init_urandom():
-        """NOT_RPYTHON
-        """
         return None
 
     def urandom(context, n):
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,5 +1,5 @@
 import sys, os
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, not_rpython
 from rpython.rlib import jit, rposix, rgc
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
@@ -35,8 +35,9 @@
 
     use_weaklist = True # False for tests
 
+    @not_rpython
     def __init__(self):
-        "NOT_RPYTHON: use _get_vmprof()"
+        "use _get_vmprof()"
         self._code_classes = set()
         self._gather_all_code_objs = lambda: None
         self._cleanup_()
@@ -66,8 +67,9 @@
             elif self.use_weaklist:
                 code._vmprof_weak_list.add_handle(code)
 
+    @not_rpython
     def register_code_object_class(self, CodeClass, full_name_func):
-        """NOT_RPYTHON
+        """
         Register statically the class 'CodeClass' as containing user
         code objects.
 
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -35,7 +35,7 @@
 # return value of tell(), but not as argument to read().
 
 import os, sys, errno
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, not_rpython
 from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib import rposix, nonconst, _rsocket_rffi as _c
 from rpython.rlib.rstring import StringBuilder
@@ -394,8 +394,8 @@
 class MMapFile(Stream):
     """Standard I/O basis stream using mmap."""
 
+    @not_rpython
     def __init__(self, fd, mmapaccess):
-        """NOT_RPYTHON"""
         self.fd = fd
         self.access = mmapaccess
         self.pos = 0
@@ -1157,8 +1157,8 @@
 class DecodingInputFilter(Stream):
     """Filtering input stream that decodes an encoded file."""
 
+    @not_rpython
     def __init__(self, base, encoding="utf8", errors="strict"):
-        """NOT_RPYTHON"""
         self.base = base
         self.do_read = base.read
         self.encoding = encoding
@@ -1203,8 +1203,8 @@
 class EncodingOutputFilter(Stream):
     """Filtering output stream that writes to an encoded file."""
 
+    @not_rpython
     def __init__(self, base, encoding="utf8", errors="strict"):
-        """NOT_RPYTHON"""
         self.base = base
         self.do_write = base.write
         self.encoding = encoding

From pypy.commits at gmail.com  Mon Nov  7 14:48:26 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 07 Nov 2016 11:48:26 -0800 (PST)
Subject: [pypy-commit] pypy default: trivial fixes for win32/MSVC
Message-ID: <5820da8a.e8edc20a.1018a.7042@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88195:a6977031946c
Date: 2016-11-07 21:45 +0200
http://bitbucket.org/pypy/pypy/changeset/a6977031946c/

Log:	trivial fixes for win32/MSVC

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -323,7 +323,7 @@
         # the stacktrace
         f = open(self.temppath, 'w')
         f.close()
-        co = compile('open("%s")' % self.temppath, self.temppath, 'exec')
+        co = compile('open(r"%s")' % self.temppath, self.temppath, 'exec')
         sys.pypy_set_track_resources(True)
         try:
             # this exec used to fail, because space.format_traceback tried to
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2315,7 +2315,7 @@
 
 _pipe2_syscall = ENoSysCache()
 
-post_include_bits=['int _cpu_count(void);']
+post_include_bits=['RPY_EXTERN int _cpu_count(void);']
 # cpu count for linux, windows and mac (+ bsds)
 # note that the code is copied from cpython and split up here
 if sys.platform.startswith('linux'):

From pypy.commits at gmail.com  Tue Nov  8 02:55:26 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Mon, 07 Nov 2016 23:55:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix most test_ttk failures
Message-ID: <582184ee.c4811c0a.655bc.74ac@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88198:d7acab6ea3f1
Date: 2016-11-08 08:54 +0100
http://bitbucket.org/pypy/pypy/changeset/d7acab6ea3f1/

Log:	Fix most test_ttk failures

diff --git a/lib-python/3/tkinter/test/test_ttk/test_extensions.py b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
--- a/lib-python/3/tkinter/test/test_ttk/test_extensions.py
+++ b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
@@ -2,7 +2,7 @@
 import unittest
 import tkinter
 from tkinter import ttk
-from test.support import requires, run_unittest, swap_attr
+from test.support import requires, run_unittest, swap_attr, gc_collect
 from tkinter.test.support import AbstractTkTest, destroy_default_root
 
 requires('gui')
diff --git a/lib-python/3/tkinter/test/test_ttk/test_widgets.py b/lib-python/3/tkinter/test/test_ttk/test_widgets.py
--- a/lib-python/3/tkinter/test/test_ttk/test_widgets.py
+++ b/lib-python/3/tkinter/test/test_ttk/test_widgets.py
@@ -1,7 +1,7 @@
 import unittest
 import tkinter
 from tkinter import ttk, TclError
-from test.support import requires
+from test.support import requires, gc_collect
 import sys
 
 from tkinter.test.test_ttk.test_functions import MockTclObj
@@ -837,6 +837,7 @@
         self.assertEqual(conv(self.scale.get()), var.get())
         self.assertEqual(conv(self.scale.get()), max + 5)
         del var
+        gc_collect()
 
         # the same happens with the value option
         self.scale['value'] = max + 10

From pypy.commits at gmail.com  Tue Nov  8 02:55:23 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Mon, 07 Nov 2016 23:55:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test_tcl
Message-ID: <582184eb.0777c20a.532be.2a01@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88196:dd00f854111a
Date: 2016-11-07 21:41 +0100
http://bitbucket.org/pypy/pypy/changeset/dd00f854111a/

Log:	Fix test_tcl

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -359,7 +359,7 @@
             for i in range(objc[0]):
                 result.append(FromObj(self, objv[0][i]))
             return tuple(result)
-        elif isinstance(arg, tuple):
+        elif isinstance(arg, (tuple, list)):
             return self._splitObj(arg)
         if isinstance(arg, str):
             arg = arg.encode('utf-8')
@@ -378,6 +378,8 @@
             return tuple(result)
         elif isinstance(arg, tuple):
             return arg
+        elif isinstance(arg, list):
+            return tuple(arg)
         elif isinstance(arg, str):
             arg = arg.encode('utf8')
 
@@ -411,6 +413,9 @@
                 result[i] = newelem
             if result is not None:
                 return tuple(result)
+        if isinstance(arg, list):
+            # Recursively invoke SplitObj for all list items.
+            return tuple(self._splitObj(elem) for elem in arg)
         elif isinstance(arg, str):
             argc = tkffi.new("int*")
             argv = tkffi.new("char***")
@@ -483,7 +488,10 @@
     def getboolean(self, s):
         if isinstance(s, int):
             return bool(s)
-        s = s.encode('utf-8')
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
+            raise TypeError
         if b'\x00' in s:
             raise TypeError
         v = tkffi.new("int*")
@@ -493,9 +501,12 @@
         return bool(v[0])
 
     def getint(self, s):
-        if isinstance(s, (int, long)):
+        if isinstance(s, int):
             return s
-        s = s.encode('utf-8')
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
+            raise TypeError
         if b'\x00' in s:
             raise TypeError
         if tklib.HAVE_LIBTOMMATH or tklib.HAVE_WIDE_INT_TYPE:
@@ -517,11 +528,14 @@
             return v[0]
 
     def getdouble(self, s):
-        if isinstance(s, float):
-            return s
-        if '\x00' in s:
+        if isinstance(s, (float, int)):
+            return float(s)
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
             raise TypeError
-        s = s.encode('utf-8')
+        if b'\x00' in s:
+            raise TypeError
         v = tkffi.new("double*")
         res = tklib.Tcl_GetDouble(self.interp, s, v)
         if res == tklib.TCL_ERROR:
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -70,7 +70,7 @@
 
 def AsBignumObj(value):
     sign = -1 if value < 0 else 1
-    hexstr = '%x' % abs(value)
+    hexstr = b'%x' % abs(value)
     bigValue = tkffi.new("mp_int*")
     tklib.mp_init(bigValue)
     try:
@@ -133,7 +133,7 @@
 
 def AsObj(value):
     if isinstance(value, bytes):
-        return tklib.Tcl_NewStringObj(value, len(value))
+        return tklib.Tcl_NewByteArrayObj(value, len(value))
     if isinstance(value, bool):
         return tklib.Tcl_NewBooleanObj(value)
     if isinstance(value, int):
@@ -151,7 +151,7 @@
                 return AsBignumObj(value)
     if isinstance(value, float):
         return tklib.Tcl_NewDoubleObj(value)
-    if isinstance(value, tuple):
+    if isinstance(value, (tuple, list)):
         argv = tkffi.new("Tcl_Obj*[]", len(value))
         for i in range(len(value)):
             argv[i] = AsObj(value[i])

From pypy.commits at gmail.com  Tue Nov  8 02:55:25 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Mon, 07 Nov 2016 23:55:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Some gc_collect() to fix test_tk
Message-ID: <582184ed.4a6ec20a.b8d27.22d5@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88197:73939029ad51
Date: 2016-11-07 21:46 +0100
http://bitbucket.org/pypy/pypy/changeset/73939029ad51/

Log:	Some gc_collect() to fix test_tk

diff --git a/lib-python/3/tkinter/test/test_tkinter/test_images.py b/lib-python/3/tkinter/test/test_tkinter/test_images.py
--- a/lib-python/3/tkinter/test/test_tkinter/test_images.py
+++ b/lib-python/3/tkinter/test/test_tkinter/test_images.py
@@ -36,6 +36,7 @@
         self.assertEqual(image.height(), 16)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def test_create_from_data(self):
@@ -50,6 +51,7 @@
         self.assertEqual(image.height(), 16)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def assertEqualStrList(self, actual, expected):
@@ -130,6 +132,7 @@
         self.assertEqual(image['file'], testfile)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def check_create_from_data(self, ext):
@@ -147,6 +150,7 @@
         self.assertEqual(image['file'], '')
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def test_create_from_ppm_file(self):

From pypy.commits at gmail.com  Tue Nov  8 03:17:12 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 08 Nov 2016 00:17:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: __qualname__ should not use the parent
 scope for "global" functions.
Message-ID: <58218a08.c32d1c0a.e86a1.7e12@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88199:9da9ea8cb259
Date: 2016-11-08 09:16 +0100
http://bitbucket.org/pypy/pypy/changeset/9da9ea8cb259/

Log:	__qualname__ should not use the parent scope for "global" functions.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -210,7 +210,9 @@
 
     def sub_scope(self, kind, name, node, lineno):
         """Convenience function for compiling a sub scope."""
-        if self.qualname:
+        if self.scope.lookup(name) == symtable.SCOPE_GLOBAL_EXPLICIT:
+            qualname = name
+        elif self.qualname:
             qualname = '%s.%s' % (self.qualname, name)
         else:
             qualname = name
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -38,6 +38,17 @@
                 pass
         assert A.f.__qualname__ == 'test_qualname_method.<locals>.A.f'
 
+    def test_qualname_global(self):
+        def f():
+            global inner_global
+            def inner_global():
+                def inner_function2():
+                    pass
+                return inner_function2
+            return inner_global
+        assert f().__qualname__ == 'inner_global'
+        assert f()().__qualname__ == 'inner_global.<locals>.inner_function2'
+
     def test_annotations(self):
         def f(): pass
         ann = f.__annotations__

From pypy.commits at gmail.com  Tue Nov  8 03:53:27 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 00:53:27 -0800 (PST)
Subject: [pypy-commit] pypy default: mark some vector tests x86 only (test
 cases are written for x86 ISA assumptions)
Message-ID: <58219287.12111c0a.2882f.8921@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88200:8017949729a8
Date: 2016-11-08 09:52 +0100
http://bitbucket.org/pypy/pypy/changeset/8017949729a8/

Log:	mark some vector tests x86 only (test cases are written for x86 ISA
	assumptions)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -418,9 +418,7 @@
     def test_sum_float_to_int16(self):
         result = self.run("sum_float_to_int16")
         assert result == sum(range(30))
-        # one can argue that this is not desired,
-        # but unpacking exactly hits savings = 0
-        self.check_vectorized(1, 1)
+
     def define_sum_float_to_int32():
         return """
         a = |30|
@@ -429,7 +427,6 @@
     def test_sum_float_to_int32(self):
         result = self.run("sum_float_to_int32")
         assert result == sum(range(30))
-        self.check_vectorized(1, 1)
 
     def define_sum_float_to_float32():
         return """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -1,6 +1,7 @@
 import py
 import sys
 import pytest
+import platform
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
@@ -125,6 +126,7 @@
         """, False)
         self.assert_equal(loop2, loop3)
 
+    @py.test.mark.skipif("not platform.machine().startswith('x86')")
     def test_int_to_float(self):
         loop1 = self.parse_trace("""
         i10 = raw_load_i(p0, i0, descr=long)
@@ -227,6 +229,7 @@
         """, False, additional_args=['v10[2xi64]'])
         self.assert_equal(loop2, loop3)
 
+    @py.test.mark.skipif("not platform.machine().startswith('x86')")
     def test_cast_float_to_int(self):
         loop1 = self.parse_trace("""
         f10 = raw_load_f(p0, i1, descr=double)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -3,7 +3,7 @@
 import pytest
 import math
 import functools
-from hypothesis import given, note, strategies as st
+from hypothesis import given, note, strategies as st, settings
 from rpython.jit.metainterp.warmspot import ll_meta_interp, get_stats
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.jit.codewriter.policy import StopAtXPolicy
@@ -160,9 +160,7 @@
                 i += size
 
         la = data.draw(st.lists(st.floats(), min_size=10, max_size=150))
-        #la = [0.0, 0.0, 0.0, 0.0, 5e-324, 0.0, 0.0, 5e-324, 0.0, 0.0]
         l = len(la)
-        #lb = [0.0] * l
         lb = data.draw(st.lists(st.floats(), min_size=l, max_size=l))
 
         rawstorage = RawStorage()
@@ -191,15 +189,14 @@
         vec_float_binary(lambda a,b: a-b, rffi.DOUBLE)
     test_vec_float_mul = \
         vec_float_binary(lambda a,b: a*b, rffi.DOUBLE)
-    #test_vec_float_div = \
-    #    vec_float_binary(lambda a,b: a/b, rffi.DOUBLE)
 
     test_vec_float_cmp_eq = \
         vec_float_binary(lambda a,b: a == b, rffi.DOUBLE)
     test_vec_float_cmp_ne = \
         vec_float_binary(lambda a,b: a != b, rffi.DOUBLE)
 
-    def _vector_simple_int(self, func, type, strat, data):
+    def _vector_simple_int(self, func, type, la):
+        oldfunc = func
         func = always_inline(func)
 
         size = rffi.sizeof(type)
@@ -214,9 +211,8 @@
                 raw_storage_setitem(vc, i, rffi.cast(type,c))
                 i += size
 
-        la = data.draw(st.lists(strat, min_size=10, max_size=150))
         l = len(la)
-        lb = data.draw(st.lists(strat, min_size=l, max_size=l))
+        lb = list(reversed(la))[:]
 
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
@@ -226,7 +222,7 @@
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
-            assert rffi.cast(type, func(la[i], lb[i])) == c
+            assert rffi.cast(type, oldfunc(la[i], lb[i])) == c
 
         rawstorage.clear()
 
@@ -235,50 +231,52 @@
         bits = size*8
         assert 0 <= bits <= 64
         integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1)
-        return pytest.mark.parametrize('func,type,strat', [
-            (arith_func, type, integers)
-        ])(given(data=st.data())(test_func))
+        @given(st.lists(integers, min_size=10, max_size=15))
+        @settings(max_examples=20)
+        def tf(self, la):
+            return test_func(self, arith_func, type, la)
+        return tf
 
     vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int)
 
-    test_vec_signed_add = \
+    test_vec_simple_int_signed_add = \
         vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED)
-    test_vec_int_add = \
-        vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT)
-    test_vec_short_add = \
-        vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT)
+    test_vec_simple_int_int_add = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)+r_int(b)), rffi.INT)
+    test_vec_simple_int_short_add = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)+r_int(b)), rffi.SHORT)
 
-    test_vec_signed_sub = \
+    test_vec_simple_int_signed_sub = \
         vec_int_arith(lambda a,b: intmask(a-b), rffi.SIGNED)
-    test_vec_sub_int = \
-        vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT)
-    test_vec_sub_short = \
-        vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT)
+    test_vec_simple_int_sub_int = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)-r_int(b)), rffi.INT)
+    test_vec_simple_int_sub_short = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)-r_int(b)), rffi.SHORT)
 
-    test_vec_signed_and = \
+    test_vec_simple_int_signed_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SIGNED)
-    test_vec_int_and = \
+    test_vec_simple_int_int_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.INT)
-    test_vec_short_and = \
+    test_vec_simple_int_short_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SHORT)
 
-    test_vec_or_signed = \
+    test_vec_simple_int_or_signed = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SIGNED)
-    test_vec_or_int = \
+    test_vec_simple_int_or_int = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.INT)
-    test_vec_or_short = \
+    test_vec_simple_int_or_short = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SHORT)
 
-    test_vec_xor_signed = \
+    test_vec_simple_int_xor_signed = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SIGNED)
-    test_vec_xor_int = \
+    test_vec_simple_int_xor_int = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.INT)
-    test_vec_xor_short = \
+    test_vec_simple_int_xor_short = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT)
 
-    test_vec_int_cmp_eq = \
+    test_vec_simple_int_int_cmp_eq = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
-    test_vec_int_cmp_ne = \
+    test_vec_simple_int_int_cmp_ne = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
 
     @py.test.mark.parametrize('i',[1,2,3,4,9])
@@ -424,9 +422,9 @@
 
     test_vec_int_sum = vec_reduce(st.integers(min_value=-2**(64-1), max_value=2**(64-1)-1),
                              lambda a,b: lltype.intmask(lltype.intmask(a)+lltype.intmask(b)), lltype.Signed)
-    test_vec_float_sum = vec_reduce(st.floats(), lambda a,b: a+b, rffi.DOUBLE)
-    test_vec_float_prod = vec_reduce(st.floats(min_value=-100, max_value=100,
-                                               allow_nan=False, allow_infinity=False), lambda a,b: a*b, rffi.DOUBLE)
+    small_floats = st.floats(min_value=-100, max_value=100, allow_nan=False, allow_infinity=False)
+    test_vec_float_sum = vec_reduce(small_floats, lambda a,b: a+b, rffi.DOUBLE)
+    test_vec_float_prod = vec_reduce(small_floats, lambda a,b: a*b, rffi.DOUBLE)
 
 
     def test_constant_expand(self):

From pypy.commits at gmail.com  Tue Nov  8 04:59:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 01:59:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix docstring
Message-ID: <5821a1e5.0a74c20a.acd08.acde@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88201:1773719613f5
Date: 2016-11-08 10:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1773719613f5/

Log:	Fix docstring

diff --git a/rpython/jit/tl/jittest.py b/rpython/jit/tl/jittest.py
--- a/rpython/jit/tl/jittest.py
+++ b/rpython/jit/tl/jittest.py
@@ -1,6 +1,6 @@
 """
 This file is imported by rpython.translator.driver when running the
-target --jittest.  Feel free to hack it as needed; it is imported
+target --pyjittest.  Feel free to hack it as needed; it is imported
 only after the '---> Checkpoint' fork.
 """
 

From pypy.commits at gmail.com  Tue Nov  8 04:59:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 01:59:02 -0800 (PST)
Subject: [pypy-commit] pypy default: One more try at tweaking this code.
 Need to turn the RPython exception
Message-ID: <5821a1e6.a115c20a.26a18.5152@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88202:474587286cd1
Date: 2016-11-08 10:58 +0100
http://bitbucket.org/pypy/pypy/changeset/474587286cd1/

Log:	One more try at tweaking this code. Need to turn the RPython
	exception into SystemError in gateway.py too, so that it can be
	caught and processed in the topmost frame (code like try: ... except
	SystemError: ... would miss it).

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -509,3 +509,28 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def _convert_unexpected_exception_extra(space, e):
+    "NOT_RPYTHON"
+    if e.__class__.__name__ in (
+        'Skipped',     # list of exception class names that are ok
+        ):             # to get during ==untranslated tests== only
+        raise
+    # include the RPython-level traceback
+    exc = sys.exc_info()
+    import traceback, cStringIO
+    f = cStringIO.StringIO()
+    print >> f, "\nTraceback (interpreter-level):"
+    traceback.print_tb(exc[2], file=f)
+    return f.getvalue()
+
+def get_converted_unexpected_exception(space, e):
+    if we_are_translated():
+        from rpython.rlib.debug import debug_print_traceback
+        debug_print_traceback()
+        extra = '; internal traceback was dumped to stderr'
+    else:
+        extra = _convert_unexpected_exception_extra(space, e)
+    return OperationError(space.w_SystemError, space.wrap(
+        "unexpected internal exception (please report a bug): %r%s" %
+        (e, extra)))
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -725,13 +725,8 @@
         except RuntimeError:   # not on top of py.py
             raise OperationError(space.w_RuntimeError, space.w_None)
         except Exception as e:      # general fall-back
-            if we_are_translated():
-                from rpython.rlib.debug import debug_print_traceback
-                debug_print_traceback()
-            # propagate the exception anyway, which will be turned
-            # into a proper OperationError(SystemError) when we
-            # reach PyFrame.execute_frame()
-            raise
+            from pypy.interpreter import error
+            raise error.get_converted_unexpected_exception(space, e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -889,30 +889,10 @@
             frame = frame.f_backref()
         return None
 
-    def _convert_unexpected_exception_extra(self, e):
-        "NOT_RPYTHON"
-        if e.__class__.__name__ in (
-            'Skipped',     # list of exception class names that are ok
-            ):             # to get during ==untranslated tests== only
-            raise
-        # include the RPython-level traceback
-        exc = sys.exc_info()
-        import traceback, cStringIO
-        f = cStringIO.StringIO()
-        print >> f, "\nTraceback (interpreter-level):"
-        traceback.print_tb(exc[2], file=f)
-        return f.getvalue()
+    def _convert_unexpected_exception(self, e):
+        from pypy.interpreter import error
 
-    def _convert_unexpected_exception(self, e):
-        if we_are_translated():
-            from rpython.rlib.debug import debug_print_traceback
-            debug_print_traceback()
-            extra = '; internal traceback(s) were dumped to stderr'
-        else:
-            extra = self._convert_unexpected_exception_extra(e)
-        operr = OperationError(self.space.w_SystemError, self.space.wrap(
-            "unexpected internal exception (please report a bug): %r%s" %
-            (e, extra)))
+        operr = error.get_converted_unexpected_exception(self.space, e)
         pytraceback.record_application_traceback(
             self.space, operr, self, self.last_instr)
         raise operr
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -787,6 +787,23 @@
         assert ('unexpected internal exception (please '
                 'report a bug): UnexpectedException') in err
 
+    def test_system_error_2(self):
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_msg = space.appexec([w_g], """(my_g):
+            try:
+                my_g()
+            except SystemError as e:
+                return str(e)
+        """)
+        err = space.str_w(w_msg)
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist

From pypy.commits at gmail.com  Tue Nov  8 05:33:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 02:33:14 -0800 (PST)
Subject: [pypy-commit] pypy default: A further tweak: move the logic
 detecting MemoryError and stack
Message-ID: <5821a9ea.28c9c20a.cb9c0.688c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88203:a4d8923ea51b
Date: 2016-11-08 11:32 +0100
http://bitbucket.org/pypy/pypy/changeset/a4d8923ea51b/

Log:	A further tweak: move the logic detecting MemoryError and stack
	overflows in error.py, where it can be used for pyopcode.py too

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,6 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib import rstackovf
 
 from pypy.interpreter import debug
 
@@ -524,13 +525,37 @@
     traceback.print_tb(exc[2], file=f)
     return f.getvalue()
 
+ at jit.dont_look_inside
 def get_converted_unexpected_exception(space, e):
-    if we_are_translated():
-        from rpython.rlib.debug import debug_print_traceback
-        debug_print_traceback()
-        extra = '; internal traceback was dumped to stderr'
-    else:
-        extra = _convert_unexpected_exception_extra(space, e)
-    return OperationError(space.w_SystemError, space.wrap(
-        "unexpected internal exception (please report a bug): %r%s" %
-        (e, extra)))
+    """This is used in two places when we get an non-OperationError
+    RPython exception: from gateway.py when calling an interp-level
+    function raises; and from pyopcode.py when we're exiting the
+    interpretation of the frame with an exception.  Note that it
+    *cannot* be used in pyopcode.py: that place gets a
+    ContinueRunningNormally exception from the JIT, which must not end
+    up here!
+    """
+    try:
+        if not we_are_translated():
+            raise
+        raise e
+    except KeyboardInterrupt:
+        return OperationError(space.w_KeyboardInterrupt, space.w_None)
+    except MemoryError:
+        return OperationError(space.w_MemoryError, space.w_None)
+    except rstackovf.StackOverflow as e:
+        rstackovf.check_stack_overflow()
+        return oefmt(space.w_RuntimeError,
+                     "maximum recursion depth exceeded")
+    except RuntimeError:   # not on top of py.py
+        return OperationError(space.w_RuntimeError, space.w_None)
+    except:
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
+        else:
+            extra = _convert_unexpected_exception_extra(space, e)
+        return OperationError(space.w_SystemError, space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -23,7 +23,6 @@
     DescrMismatch)
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode
-from rpython.rlib import rstackovf
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
@@ -714,16 +713,6 @@
             raise e
         except OperationError:
             raise
-        except KeyboardInterrupt:
-            raise OperationError(space.w_KeyboardInterrupt, space.w_None)
-        except MemoryError:
-            raise OperationError(space.w_MemoryError, space.w_None)
-        except rstackovf.StackOverflow as e:
-            rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RuntimeError,
-                        "maximum recursion depth exceeded")
-        except RuntimeError:   # not on top of py.py
-            raise OperationError(space.w_RuntimeError, space.w_None)
         except Exception as e:      # general fall-back
             from pypy.interpreter import error
             raise error.get_converted_unexpected_exception(space, e)

From pypy.commits at gmail.com  Tue Nov  8 05:36:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 02:36:43 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <5821aabb.61adc20a.49c2f.7f9f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88204:95700489ad41
Date: 2016-11-08 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/95700489ad41/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -418,7 +418,7 @@
         guard_no_exception(descr=...)
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(ptr85), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>)
-        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_length .+>)
+        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_allocated_length .+>)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--

From pypy.commits at gmail.com  Tue Nov  8 05:48:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 02:48:03 -0800 (PST)
Subject: [pypy-commit] pypy default: fix tests
Message-ID: <5821ad63.ca73c20a.b998c.697d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88205:4c405c663a1c
Date: 2016-11-08 11:47 +0100
http://bitbucket.org/pypy/pypy/changeset/4c405c663a1c/

Log:	fix tests

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -543,6 +543,9 @@
         return OperationError(space.w_KeyboardInterrupt, space.w_None)
     except MemoryError:
         return OperationError(space.w_MemoryError, space.w_None)
+    except NotImplementedError:   # not on top of pypy! tests only
+        return OperationError(space.w_SystemError,
+                              space.wrap("NotImplementedError"))
     except rstackovf.StackOverflow as e:
         rstackovf.check_stack_overflow()
         return oefmt(space.w_RuntimeError,
diff --git a/pypy/module/gc/test/test_app_referents.py b/pypy/module/gc/test/test_app_referents.py
--- a/pypy/module/gc/test/test_app_referents.py
+++ b/pypy/module/gc/test/test_app_referents.py
@@ -4,12 +4,13 @@
 
 def test_interface_to_dump_rpy_heap_str(space):
     filename = str(udir.join('dump_rpy_heap.str'))
-    try:
-        space.appexec([space.wrap(filename)], """(filename):
-            import gc
-            gc.dump_rpy_heap(filename)""")
-    except NotImplementedError:
-        pass
+    space.appexec([space.wrap(filename)], """(filename):
+        import gc
+        try:
+            gc.dump_rpy_heap(filename)
+        except SystemError:
+            pass
+    """)
     assert os.path.exists(filename)
 
 def test_interface_to_dump_rpy_heap_file(space):
@@ -20,20 +21,22 @@
             f.write('X')
             return f""")
     assert os.path.getsize(filename) == 0   # the 'X' was not flushed yet
-    try:
-        space.appexec([w_f], """(f):
-            import gc
-            gc.dump_rpy_heap(f)""")
-    except NotImplementedError:
-        pass
+    space.appexec([w_f], """(f):
+        import gc
+        try:
+            gc.dump_rpy_heap(f)
+        except SystemError:
+            pass
+    """)
     assert os.path.getsize(filename) == 1   # the 'X' was flushed here
 
 def test_interface_to_dump_rpy_heap_fd(space):
     filename = str(udir.join('dump_rpy_heap.fd'))
     f = open(filename, 'wb')
-    try:
-        space.appexec([space.wrap(f.fileno())], """(fd):
-            import gc
-            gc.dump_rpy_heap(fd)""")
-    except NotImplementedError:
-        pass
+    space.appexec([space.wrap(f.fileno())], """(fd):
+        import gc
+        try:
+            gc.dump_rpy_heap(fd)
+        except SystemError:
+            pass
+    """)

From pypy.commits at gmail.com  Tue Nov  8 05:50:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 02:50:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Document branch
Message-ID: <5821ae06.13d41c0a.1182.b9bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88206:093f41c167de
Date: 2016-11-08 11:49 +0100
http://bitbucket.org/pypy/pypy/changeset/093f41c167de/

Log:	Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,8 @@
 .. this is a revision shortly after release-pypy2.7-v5.6
 .. startrev: 7e9787939641
 
+.. branch: rpython-error-to-systemerror
 
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.

From pypy.commits at gmail.com  Tue Nov  8 07:12:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:12:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge 30e64025212d (from openssl-1.1)
Message-ID: <5821c12f.0209c20a.36aab.9233@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88207:a4c50a3fcf9f
Date: 2016-11-08 13:11 +0100
http://bitbucket.org/pypy/pypy/changeset/a4c50a3fcf9f/

Log:	hg merge 30e64025212d (from openssl-1.1)

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names
+from pypy.module._hashlib.interp_hashlib import (
+    algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC)
 
 
 class Module(MixedModule):
@@ -13,6 +14,9 @@
     for name in algorithms:
         interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
 
+    if HAS_FAST_PKCS5_PBKDF2_HMAC:
+        interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac'
+
     def startup(self, space):
         w_meth_names = fetch_names(space)
         space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -8,7 +8,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.thread.os_lock import Lock
 
@@ -58,23 +58,26 @@
     def __init__(self, space, name, copy_from=NULL_CTX):
         self.name = name
         digest_type = self.digest_type_by_name(space)
-        self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
+        self.digest_size = ropenssl.EVP_MD_size(digest_type)
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
-        ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
         rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
         try:
             if copy_from:
-                ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
+                    raise ValueError
             else:
                 ropenssl.EVP_DigestInit(ctx, digest_type)
             self.ctx = ctx
         except:
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
             raise
         self.register_finalizer(space)
 
@@ -82,8 +85,7 @@
         ctx = self.ctx
         if ctx:
             self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-            ropenssl.EVP_MD_CTX_cleanup(ctx)
-            lltype.free(ctx, flavor='raw')
+            ropenssl.EVP_MD_CTX_free(ctx)
 
     def digest_type_by_name(self, space):
         digest_type = ropenssl.EVP_get_digestbyname(self.name)
@@ -128,21 +130,26 @@
 
     def get_block_size(self, space):
         digest_type = self.digest_type_by_name(space)
-        block_size = rffi.getintfield(digest_type, 'c_block_size')
+        block_size = ropenssl.EVP_MD_block_size(digest_type)
         return space.wrap(block_size)
 
     def get_name(self, space):
         return space.wrap(self.name)
 
     def _digest(self, space):
-        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+        ctx = ropenssl.EVP_MD_CTX_new()
+        if ctx is None:
+            raise MemoryError
+        try:
             with self.lock:
-                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+                if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx):
+                    raise ValueError
             digest_size = self.digest_size
             with rffi.scoped_alloc_buffer(digest_size) as buf:
                 ropenssl.EVP_DigestFinal(ctx, buf.raw, None)
-                ropenssl.EVP_MD_CTX_cleanup(ctx)
                 return buf.str(digest_size)
+        finally:
+            ropenssl.EVP_MD_CTX_free(ctx)
 
 
 W_Hash.typedef = TypeDef(
@@ -176,3 +183,27 @@
 for _name in algorithms:
     _newname = 'new_%s' % (_name,)
     globals()[_newname] = make_new_hash(_name, _newname)
+
+
+HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
+if HAS_FAST_PKCS5_PBKDF2_HMAC:
+    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+                 w_dklen=WrappedDefault(None))
+    def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
+        digest = ropenssl.EVP_get_digestbyname(name)
+        if not digest:
+            raise oefmt(space.w_ValueError, "unknown hash function")
+        if space.is_w(w_dklen, space.w_None):
+            dklen = ropenssl.EVP_MD_size(digest)
+        else:
+            dklen = space.int_w(w_dklen)
+        if dklen < 1:
+            raise oefmt(space.w_ValueError,
+                        "key length must be greater than 0.")
+        with rffi.scoped_alloc_buffer(dklen) as buf:
+            r = ropenssl.PKCS5_PBKDF2_HMAC(
+                password, len(password), salt, len(salt), rounds, digest,
+                dklen, buf.raw)
+            if not r:
+                raise ValueError
+            return space.wrap(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -15,16 +15,18 @@
 
     def test_attributes(self):
         import hashlib
-        for name, expected_size in {'md5': 16,
-                                    'sha1': 20,
-                                    'sha224': 28,
-                                    'sha256': 32,
-                                    'sha384': 48,
-                                    'sha512': 64,
-                                    }.items():
+        for name, (expected_size, expected_block_size) in {
+                'md5': (16, 64),
+                'sha1': (20, 64),
+                'sha224': (28, 64),
+                'sha256': (32, 64),
+                'sha384': (48, 128),
+                'sha512': (64, 128),
+                }.items():
             h = hashlib.new(name)
             assert h.name == name
             assert h.digest_size == expected_size
+            assert h.block_size == expected_block_size
             #
             h.update(b'abc')
             h2 = h.copy()
@@ -83,3 +85,13 @@
             assert got and type(got) is str and len(got) % 2 == 0
             if expected is not None:
                 assert got == expected
+
+    def test_pbkdf2(self):
+        try:
+            from _hashlib import pbkdf2_hmac
+        except ImportError:
+            skip("Requires OpenSSL >= 1.1")
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
+        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -37,9 +37,10 @@
 PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
 
 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
+ PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
  PY_SSL_VERSION_TLS1_2) = range(6)
 
+
 SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
 SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
@@ -72,11 +73,12 @@
 constants["HAS_NPN"] = HAS_NPN
 constants["HAS_ALPN"] = HAS_ALPN
 
+constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
+constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 if not OPENSSL_NO_SSL3:
     constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 if HAVE_TLSv1_2:
     constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
@@ -117,7 +119,10 @@
         err_reason = libssl_ERR_GET_REASON(errcode)
         reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
         lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        msg = rffi.charp2str(libssl_ERR_reason_error_string(errcode))
+        raw_msg = libssl_ERR_reason_error_string(errcode)
+        msg = None
+        if raw_msg:
+            msg = rffi.charp2str(raw_msg)
     if not msg:
         msg = "unknown error"
     if reason_str and lib_str:
@@ -650,9 +655,12 @@
         if not self.ssl:
             return space.w_None
         comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method or intmask(comp_method[0].c_type) == NID_undef:
+        if not comp_method:
             return space.w_None
-        short_name = libssl_OBJ_nid2sn(comp_method[0].c_type)
+        method_type = intmask(libssl_COMP_get_type(comp_method))
+        if method_type == NID_undef:
+            return space.w_None
+        short_name = libssl_COMP_get_name(comp_method)
         if not short_name:
             return space.w_None
         return space.wrap(rffi.charp2str(short_name))
@@ -819,7 +827,7 @@
     for index in range(entry_count):
         entry = libssl_X509_NAME_get_entry(xname, index)
         # check to see if we've gotten to a new RDN
-        entry_level = intmask(entry[0].c_set)
+        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
         if rdn_level >= 0:
             if rdn_level != entry_level:
                 # yes, new RDN
@@ -870,8 +878,9 @@
                                 "No method for internalizing subjectAltName!'")
 
             with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                p_ptr[0] = ext[0].c_value.c_data
-                length = intmask(ext[0].c_value.c_length)
+                ext_value = libssl_X509_EXTENSION_get_data(ext)
+                p_ptr[0] = ext_value.c_data
+                length = intmask(ext_value.c_length)
                 null = lltype.nullptr(rffi.VOIDP.TO)
                 if method[0].c_it:
                     names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
@@ -991,10 +1000,8 @@
     if OPENSSL_VERSION_NUMBER >= 0x10001000:
         # Calls x509v3_cache_extensions and sets up crldp
         libssl_X509_check_ca(certificate)
-        dps = certificate[0].c_crldp
-    else:
-        dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-            certificate, NID_crl_distribution_points, None, None))
+    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
+        certificate, NID_crl_distribution_points, None, None))
     if not dps:
         return None
 
@@ -1337,14 +1344,14 @@
     ctx = lltype.nullptr(SSL_CTX.TO)
 
     def __init__(self, space, protocol):
-        if protocol == PY_SSL_VERSION_TLS1:
+        if protocol == PY_SSL_VERSION_TLS:
+            method = libssl_TLS_method()
+        elif protocol == PY_SSL_VERSION_TLS1:
             method = libssl_TLSv1_method()
         elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
             method = libssl_SSLv3_method()
         elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
             method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_SSL23:
-            method = libssl_SSLv23_method()
         elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
             method = libssl_TLSv1_1_method()
         elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
@@ -1374,6 +1381,7 @@
             # OpenSSL 1.0.2+), or use prime256v1 by default.
             # This is Apache mod_ssl's initialization
             # policy, so we should be safe.
+            # OpenSSL 1.1 has it enabled by default.
             if libssl_SSL_CTX_set_ecdh_auto:
                 libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
@@ -1393,7 +1401,7 @@
 
     @staticmethod
     @unwrap_spec(protocol=int)
-    def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_SSL23):
+    def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_TLS):
         self = space.allocate_instance(SSLContext, w_subtype)
         self.__init__(space, protocol)
         return space.wrap(self)
@@ -1485,20 +1493,22 @@
 
     def descr_get_verify_flags(self, space):
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         return space.wrap(flags)
 
     def descr_set_verify_flags(self, space, w_obj):
         new_flags = space.int_w(w_obj)
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
+        param = libssl_X509_STORE_get0_param(store)
+        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
         flags_clear = flags & ~new_flags
         flags_set = ~flags & new_flags
         if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                store[0].c_param, flags_clear):
+                param, flags_clear):
             raise _ssl_seterror(space, None, 0)
         if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                store[0].c_param, flags_set):
+                param, flags_set):
             raise _ssl_seterror(space, None, 0)
 
     def descr_get_check_hostname(self, space):
@@ -1712,14 +1722,16 @@
         x509 = 0
         x509_ca = 0
         crl = 0
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) == X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
+            if obj_type == X509_LU_X509:
                 x509 += 1
                 if libssl_X509_check_ca(
-                        libssl_pypy_X509_OBJECT_data_x509(obj)):
+                        libssl_X509_OBJECT_get0_X509(obj)):
                     x509_ca += 1
-            elif intmask(obj.c_type) == X509_LU_CRL:
+            elif obj_type == X509_LU_CRL:
                 crl += 1
             else:
                 # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
@@ -1758,13 +1770,14 @@
             binary_mode = False
         rlist = []
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-            if intmask(obj.c_type) != X509_LU_X509:
+        objs = libssl_X509_STORE_get0_objects(store)
+        for i in range(libssl_sk_X509_OBJECT_num(objs)):
+            obj = libssl_sk_X509_OBJECT_value(objs, i)
+            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
                 # not a x509 cert
                 continue
             # CA for any purpose
-            cert = libssl_pypy_X509_OBJECT_data_x509(obj)
+            cert = libssl_X509_OBJECT_get0_X509(obj)
             if not libssl_X509_check_ca(cert):
                 continue
             if binary_mode:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -108,13 +108,13 @@
     def test_context(self):
         import _ssl, sys
         py33 = sys.version_info[:2] == (3, 3)
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         raises(ValueError, _ssl._SSLContext, -1)
 
         assert type(s.options) is int
-        assert s.options & _ssl.OP_NO_SSLv2
-        s.options &= ~_ssl.OP_NO_SSLv2
-        assert not s.options & _ssl.OP_NO_SSLv2
+        assert s.options & _ssl.OP_NO_SSLv3
+        s.options &= ~_ssl.OP_NO_SSLv3
+        assert not s.options & _ssl.OP_NO_SSLv3
         raises(TypeError, "s.options = 2.5")
 
         if py33:
@@ -148,7 +148,7 @@
 
     def test_set_default_verify_paths(self):
         import _ssl
-        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         s.set_default_verify_paths()
 
 
@@ -256,7 +256,7 @@
         self.s.close()
         del ss; gc.collect()
         
-    def test_peer_certificate(self):
+    def test_peer_certificate_1(self):
         import gc, ssl
         ss = ssl.wrap_socket(self.s)
         assert ss.getpeercert() == {}
@@ -269,13 +269,44 @@
         if not _ssl.HAS_NPN:
             skip("NPN requires OpenSSL 1.0.1 or greater")
 
-        ctx = _ssl._SSLContext()
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s, True,
                               server_hostname="svn.python.org")
         self.s.close()
         del ss; gc.collect()
 
+    def test_peer_certificate(self):
+        import _ssl, gc
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ss = ctx._wrap_socket(self.s, False)
+        ss.do_handshake()
+        assert isinstance(ss.peer_certificate(der=True), bytes)
+        assert isinstance(ss.peer_certificate(), dict)
+        self.s.close()
+        del ss; gc.collect()
+
+    def test_peer_certificate_verify(self):
+        import _ssl, ssl, gc
+        paths = ssl.get_default_verify_paths()
+        if not paths.capath and not paths.cafile:
+            skip("ssl.get_default_verify_paths() failed to return any path")
+
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.verify_mode = _ssl.CERT_REQUIRED
+        ctx.load_verify_locations(capath=paths.capath, cafile=paths.cafile)
+
+        ss = ctx._wrap_socket(self.s, False)
+        try:
+            ss.do_handshake()
+        except _ssl.SSLError as e:
+            if e.reason == 'CERTIFICATE_VERIFY_FAILED':
+                skip("Certificate verification failed. "
+                     "Most likely we just don't have any CA certificates.")
+        assert ss.peer_certificate()
+        self.s.close()
+        del ss; gc.collect()
+
     def test_tls_unique_cb(self):
         import ssl, sys, gc
         ss = ssl.wrap_socket(self.s)
@@ -340,7 +371,7 @@
 
     def test_load_cert_chain(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
         exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
@@ -359,11 +390,11 @@
 
     def test_load_verify_locations(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         ctx.load_verify_locations(cafile=self.keycert, capath=None)
 
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         with open(self.keycert) as f:
             cacert_pem = f.read()
         ctx.load_verify_locations(cadata=cacert_pem)
@@ -381,8 +412,7 @@
 
     def test_get_ca_certs(self):
         import _ssl
-
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_verify_locations(self.keycert)
         assert ctx.get_ca_certs() == []
         ctx.load_verify_locations(self.python_org_cert)
@@ -396,7 +426,7 @@
 
     def test_cert_store_stats(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
         ctx.load_cert_chain(self.keycert)
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0}
@@ -405,7 +435,7 @@
 
     def test_load_dh_params(self):
         import _ssl, errno
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
@@ -415,7 +445,7 @@
 
     def test_set_ecdh_curve(self):
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         ctx.set_ecdh_curve("prime256v1")
         raises(ValueError, ctx.set_ecdh_curve, "foo")
 
@@ -461,7 +491,7 @@
     def test_lib_reason(self):
         # Test the library and reason attributes
         import _ssl
-        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
         exc = raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
         assert exc.value.library == 'PEM'
         assert exc.value.reason == 'NO_START_LINE'
@@ -473,7 +503,7 @@
         import socket
         # Check that the appropriate SSLError subclass is raised
         # (this only tests one of them)
-        ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         with socket.socket() as s:
             s.bind(("127.0.0.1", 0))
             s.listen(5)
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -57,8 +57,19 @@
         '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)',
         '#define pypy_GENERAL_NAME_uri(name) (name->d.uniformResourceIdentifier)',
         '#define pypy_GENERAL_NAME_pop_free(names) (sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free))',
-        '#define pypy_X509_OBJECT_data_x509(obj) (obj->data.x509)',
         '#define pypy_DIST_POINT_fullname(obj) (obj->distpoint->name.fullname)',
+        # Backwards compatibility for functions introduced in 1.1
+        '#if (OPENSSL_VERSION_NUMBER < 0x10100000) || defined(LIBRESSL_VERSION_NUMBER)\n'
+        '#  define COMP_get_name(meth) (meth->name)\n'
+        '#  define COMP_get_type(meth) (meth->type)\n'
+        '#  define X509_NAME_ENTRY_set(ne) (ne->set)\n'
+        '#  define X509_OBJECT_get0_X509(obj) (obj->data.x509)\n'
+        '#  define X509_OBJECT_get_type(obj) (obj->type)\n'
+        '#  define X509_STORE_get0_objects(store) (store->objs)\n'
+        '#  define X509_STORE_get0_param(store) (store->param)\n'
+        '#else\n'
+        '#  define OPENSSL_NO_SSL2\n'
+        '#endif',
     ],
 )
 
@@ -88,6 +99,7 @@
             "OPENSSL_EXPORT_VAR_AS_FUNCTION")
     OPENSSL_VERSION_NUMBER = rffi_platform.ConstantInteger(
         "OPENSSL_VERSION_NUMBER")
+    LIBRESSL = rffi_platform.Defined("LIBRESSL_VERSION_NUMBER")
 
 cconfig = rffi_platform.configure(CConfigBootstrap)
 if cconfig["OPENSSL_EXPORT_VAR_AS_FUNCTION"]:
@@ -95,23 +107,10 @@
 else:
     ASN1_ITEM_EXP = ASN1_ITEM
 OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
+LIBRESSL = cconfig["LIBRESSL"]
+OPENSSL_1_1 = OPENSSL_VERSION_NUMBER >= 0x10100000 and not LIBRESSL
 HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
 
-if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
-     OPENSSL_VERSION_NUMBER < 0x20000000):    # <= libressl :-(
-    eci.pre_include_bits = ()
-    eci.post_include_bits = ()
-    raise Exception("""OpenSSL version >= 1.1 not supported yet.
-
-    This program requires OpenSSL version 1.0.x, and may also
-    work with LibreSSL or OpenSSL 0.9.x.  OpenSSL 1.1 is quite
-    some work to update to; contributions are welcome.  Sorry,
-    you need to install an older version of OpenSSL for now.
-    Make sure this older version is the one picked up by this
-    program when it runs the compiler.
-    
-    This is the configuration used: %r""" % (eci,))
-
 
 class CConfig:
     _compilation_info_ = eci
@@ -210,28 +209,10 @@
     OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger(
         "OBJ_NAME_TYPE_MD_METH")
 
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        X509_st = rffi_platform.Struct(
-            'struct x509_st',
-            [('crldp', stack_st_DIST_POINT)])
-
     # Some structures, with only the fields used in the _ssl module
-    X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st',
-                                              [('set', rffi.INT)])
     asn1_string_st = rffi_platform.Struct('struct asn1_string_st',
                                           [('length', rffi.INT),
                                            ('data', rffi.CCHARP)])
-    X509_extension_st = rffi_platform.Struct(
-        'struct X509_extension_st',
-        [('value', ASN1_STRING)])
-    x509_store_st = rffi_platform.Struct(
-        'struct x509_store_st',
-        [('objs', stack_st_X509_OBJECT),
-         ('param', X509_VERIFY_PARAM)])
-    x509_object_st = rffi_platform.Struct(
-        'struct x509_object_st',
-        [('type', rffi.INT)])
-
     X509_LU_X509 = rffi_platform.ConstantInteger("X509_LU_X509")
     X509_LU_CRL = rffi_platform.ConstantInteger("X509_LU_CRL")
 
@@ -244,12 +225,6 @@
     GENERAL_NAME_st = rffi_platform.Struct(
         'struct GENERAL_NAME_st',
         [('type', rffi.INT)])
-    EVP_MD_st = rffi_platform.Struct(
-        'EVP_MD',
-        [('md_size', rffi.INT),
-         ('block_size', rffi.INT)])
-    EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
-    EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
     OBJ_NAME_st = rffi_platform.Struct(
         'OBJ_NAME',
@@ -257,10 +232,6 @@
          ('name', rffi.CCHARP),
          ])
 
-    COMP_METHOD_st = rffi_platform.Struct(
-        'struct comp_method_st',
-        [('type', rffi.INT),])
-
     ACCESS_DESCRIPTION_st = rffi_platform.Struct(
         'struct ACCESS_DESCRIPTION_st',
         [('method', ASN1_OBJECT),
@@ -275,14 +246,12 @@
 SSL_CIPHER = rffi.COpaquePtr('SSL_CIPHER')
 SSL = rffi.COpaquePtr('SSL')
 BIO = rffi.COpaquePtr('BIO')
-if OPENSSL_VERSION_NUMBER >= 0x10001000:
-    X509 = rffi.CArrayPtr(X509_st)
-else:
-    X509 = rffi.COpaquePtr('X509')
-X509_NAME_ENTRY = rffi.CArrayPtr(X509_name_entry_st)
-X509_EXTENSION = rffi.CArrayPtr(X509_extension_st)
-X509_STORE = rffi.CArrayPtr(x509_store_st)
-X509_OBJECT = lltype.Ptr(x509_object_st)
+X509 = rffi.COpaquePtr('X509')
+X509_OBJECT = rffi.COpaquePtr('X509_OBJECT')
+COMP_METHOD = rffi.COpaquePtr('COMP_METHOD')
+X509_NAME_ENTRY = rffi.COpaquePtr('X509_NAME_ENTRY')
+X509_EXTENSION = rffi.COpaquePtr('X509_EXTENSION')
+X509_STORE = rffi.COpaquePtr('X509_STORE')
 X509V3_EXT_METHOD = rffi.CArrayPtr(v3_ext_method)
 ASN1_STRING.TO.become(asn1_string_st)
 ASN1_TIME = rffi.COpaquePtr('ASN1_TIME')
@@ -290,9 +259,9 @@
 GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES')
 GENERAL_NAME.TO.become(GENERAL_NAME_st)
 OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
-COMP_METHOD = rffi.CArrayPtr(COMP_METHOD_st)
 ACCESS_DESCRIPTION = rffi.CArrayPtr(ACCESS_DESCRIPTION_st)
 
+
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_OPENSSL_FINISHED = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df and \
@@ -314,8 +283,10 @@
     globals()['libssl_' + name] = external(
         name, argtypes, restype, **kw)
 
-ssl_external('SSL_load_error_strings', [], lltype.Void)
-ssl_external('SSL_library_init', [], rffi.INT)
+ssl_external('SSL_load_error_strings', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
+ssl_external('SSL_library_init', [], rffi.INT,
+    macro=OPENSSL_1_1 or None)
 ssl_external('CRYPTO_num_locks', [], rffi.INT)
 ssl_external('CRYPTO_set_locking_callback',
              [lltype.Ptr(lltype.FuncType(
@@ -341,7 +312,11 @@
     ssl_external('TLSv1_2_method', [], SSL_METHOD)
 ssl_external('SSLv2_method', [], SSL_METHOD)
 ssl_external('SSLv3_method', [], SSL_METHOD)
-ssl_external('SSLv23_method', [], SSL_METHOD)
+# Windows note: fails in untranslated tests if the following function is
+# made 'macro=True'.  Not sure I want to dig into the reason for that mess.
+libssl_TLS_method = external(
+    'TLS_method' if OPENSSL_1_1 else 'SSLv23_method',
+    [], SSL_METHOD)
 ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT,
@@ -370,7 +345,7 @@
              lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void, macro=True)
 ssl_external('SSL_CTX_set_tmp_ecdh', [SSL_CTX, EC_KEY], lltype.Void, macro=True)
-if OPENSSL_VERSION_NUMBER >= 0x10002000:
+if OPENSSL_VERSION_NUMBER >= 0x10002000 and not OPENSSL_1_1:
     ssl_external('SSL_CTX_set_ecdh_auto', [SSL_CTX, rffi.INT], lltype.Void,
                  macro=True)
 else:
@@ -416,6 +391,8 @@
 ssl_external('X509_NAME_get_entry', [X509_NAME, rffi.INT], X509_NAME_ENTRY)
 ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT)
 ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING)
+ssl_external('X509_NAME_ENTRY_set', [X509_NAME_ENTRY], rffi.INT,
+    macro=(not OPENSSL_1_1) or None)
 ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT, save_err=SAVE_ERR)
 ssl_external('X509_free', [X509], lltype.Void, releasegil=False)
 ssl_external('X509_check_ca', [X509], rffi.INT)
@@ -427,11 +404,16 @@
 ssl_external('X509_get_ext', [X509, rffi.INT], X509_EXTENSION)
 ssl_external('X509_get_ext_d2i', [X509, rffi.INT, rffi.VOIDP, rffi.VOIDP], rffi.VOIDP)
 ssl_external('X509V3_EXT_get', [X509_EXTENSION], X509V3_EXT_METHOD)
+ssl_external('X509_EXTENSION_get_data', [X509_EXTENSION], ASN1_STRING)
 
 ssl_external('X509_VERIFY_PARAM_get_flags', [X509_VERIFY_PARAM], rffi.ULONG)
 ssl_external('X509_VERIFY_PARAM_set_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_VERIFY_PARAM_clear_flags', [X509_VERIFY_PARAM, rffi.ULONG], rffi.INT)
 ssl_external('X509_STORE_add_cert', [X509_STORE, X509], rffi.INT)
+ssl_external('X509_STORE_get0_objects', [X509_STORE], stack_st_X509_OBJECT,
+    macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_STORE_get0_param', [X509_STORE], X509_VERIFY_PARAM,
+    macro=bool(not OPENSSL_1_1) or None)
 
 ssl_external('X509_get_default_cert_file_env', [], rffi.CCHARP)
 ssl_external('X509_get_default_cert_file', [], rffi.CCHARP)
@@ -468,8 +450,14 @@
              macro=True)
 ssl_external('sk_X509_OBJECT_value', [stack_st_X509_OBJECT, rffi.INT],
              X509_OBJECT, macro=True)
-ssl_external('pypy_X509_OBJECT_data_x509', [X509_OBJECT], X509,
-             macro=True)
+ssl_external('X509_OBJECT_get0_X509', [X509_OBJECT], X509,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('X509_OBJECT_get_type', [X509_OBJECT], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_name', [COMP_METHOD], rffi.CCHARP,
+             macro=bool(not OPENSSL_1_1) or None)
+ssl_external('COMP_get_type', [COMP_METHOD], rffi.INT,
+             macro=bool(not OPENSSL_1_1) or None)
 ssl_external('sk_DIST_POINT_num', [stack_st_DIST_POINT], rffi.INT,
              macro=True)
 ssl_external('sk_DIST_POINT_value', [stack_st_DIST_POINT, rffi.INT], DIST_POINT,
@@ -511,8 +499,12 @@
 # with the GIL held, and so is allowed to run in a RPython __del__ method.
 ssl_external('SSL_free', [SSL], lltype.Void, releasegil=False)
 ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, releasegil=False)
-ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
-libssl_OPENSSL_free = libssl_CRYPTO_free
+if OPENSSL_1_1:
+    ssl_external('OPENSSL_free', [rffi.VOIDP], lltype.Void, macro=True)
+else:
+    ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void)
+    libssl_OPENSSL_free = libssl_CRYPTO_free
+    del libssl_CRYPTO_free
 
 ssl_external('SSL_write', [SSL, rffi.CCHARP, rffi.INT], rffi.INT,
              save_err=SAVE_ERR)
@@ -576,10 +568,11 @@
             SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = lltype.Ptr(EVP_MD_st)
+EVP_MD = rffi.COpaquePtr('EVP_MD')
 
 OpenSSL_add_all_digests = external(
-    'OpenSSL_add_all_digests', [], lltype.Void)
+    'OpenSSL_add_all_digests', [], lltype.Void,
+    macro=OPENSSL_1_1 or None)
 EVP_get_digestbyname = external(
     'EVP_get_digestbyname',
     [rffi.CCHARP], EVP_MD)
@@ -592,10 +585,25 @@
 EVP_DigestFinal = external(
     'EVP_DigestFinal',
     [EVP_MD_CTX, rffi.CCHARP, rffi.VOIDP], rffi.INT)
+EVP_MD_size = external(
+    'EVP_MD_size', [EVP_MD], rffi.INT)
+EVP_MD_block_size = external(
+    'EVP_MD_block_size', [EVP_MD], rffi.INT)
 EVP_MD_CTX_copy = external(
     'EVP_MD_CTX_copy', [EVP_MD_CTX, EVP_MD_CTX], rffi.INT)
-EVP_MD_CTX_cleanup = external(
-    'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, releasegil=False)
+EVP_MD_CTX_new = external(
+    'EVP_MD_CTX_new' if OPENSSL_1_1 else 'EVP_MD_CTX_create',
+    [], EVP_MD_CTX)
+EVP_MD_CTX_free = external(
+    'EVP_MD_CTX_free' if OPENSSL_1_1 else 'EVP_MD_CTX_destroy',
+    [EVP_MD_CTX], lltype.Void, releasegil=False)
+
+if OPENSSL_1_1:
+    PKCS5_PBKDF2_HMAC = external('PKCS5_PBKDF2_HMAC', [
+        rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT, rffi.INT, EVP_MD,
+        rffi.INT, rffi.CCHARP], rffi.INT)
+else:
+    PKCS5_PBKDF2_HMAC = None
 
 OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType(
         [OBJ_NAME, rffi.VOIDP], lltype.Void))
@@ -606,7 +614,7 @@
 # Used for adding memory pressure. Last number is an (under?)estimate of
 # EVP_PKEY_CTX's size.
 # XXX: Make a better estimate here
-HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \
+HASH_MALLOC_SIZE = 120 + 48 \
         + rffi.sizeof(EVP_MD) * 2 + 208
 
 def init_ssl():
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1575,15 +1575,13 @@
 
         class A:
             def __init__(self):
-                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
-                    flavor='raw')
                 digest = ropenssl.EVP_get_digestbyname('sha1')
+                self.ctx = ropenssl.EVP_MD_CTX_new()
                 ropenssl.EVP_DigestInit(self.ctx, digest)
                 rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
 
             def __del__(self):
-                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
-                lltype.free(self.ctx, flavor='raw')
+                ropenssl.EVP_MD_CTX_free(self.ctx)
         #A() --- can't call it here?? get glibc crashes on tannit64
         def f():
             am1 = am2 = am3 = None

From pypy.commits at gmail.com  Tue Nov  8 07:25:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:25:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge 50aae6a02e7f^
Message-ID: <5821c42d.6a5cc20a.7e235.e94c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88208:e8664fda6f93
Date: 2016-11-08 13:18 +0100
http://bitbucket.org/pypy/pypy/changeset/e8664fda6f93/

Log:	hg merge 50aae6a02e7f^

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -16,6 +16,9 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
+We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+well as maintaining support for previous versions.
+
 XXX
 
 As always, this release fixed many issues and bugs raised by the
@@ -72,7 +75,7 @@
 * New features
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
-  * Backport fixes to rposix on windows from py2.5
+  * Backport fixes to rposix on windows from py3.5
   * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
   * Add ``inode`` to ``scandir()`` on posix systems
   * Support more attributes on ``super``
@@ -98,6 +101,8 @@
   * Support translation on FreeBSD running on PowerPC
   * Implement ``__rmod__`` on ``str`` and ``unicode`` types
   * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -86,3 +86,20 @@
 
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: newinitwarn
+
+Issue warnings for stricter handling of ``__new/init__`` args (that
+become TypeErrors in python 3)
+
+.. branch: openssl-1.1
+
+PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
+installed.  Thanks tumbleweed!
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.8.4"
+VERSION = "1.9.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.8.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -507,6 +507,14 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recv_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recv_into(buffer, [nbytes[, flags]]) -> nbytes_read
+
+        A version of recv() that stores its data into a buffer rather than creating
+        a new string.  Receive up to buffersize bytes from the socket.  If buffersize
+        is not specified (or 0), receive up to the size available in the given buffer.
+
+        See recv() for documentation about the flags.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0 or nbytes > lgt:
@@ -518,6 +526,10 @@
 
     @unwrap_spec(nbytes=int, flags=int)
     def recvfrom_into_w(self, space, w_buffer, nbytes=0, flags=0):
+        """recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)
+
+        Like recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.
+        """
         rwbuffer = space.getarg_w('w*', w_buffer)
         lgt = rwbuffer.getlength()
         if nbytes == 0:
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -523,8 +523,8 @@
             maxindex = XML_GetSpecifiedAttributeCount(self.itself)
         else:
             maxindex = 0
-        while attrs[maxindex]:
-            maxindex += 2 # copied
+            while attrs[maxindex]:
+                maxindex += 2 # copied
 
         if self.ordered_attributes:
             w_attrs = space.newlist([
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1362,8 +1362,14 @@
 def _time_to_timeval(t, l_timeval):
     import math
     fracpart, intpart = math.modf(t)
-    rffi.setintfield(l_timeval, 'c_tv_sec', int(intpart))
-    rffi.setintfield(l_timeval, 'c_tv_usec', int(fracpart * 1e6))
+    intpart = int(intpart)
+    fracpart = int(fracpart * 1e6)
+    if fracpart < 0:
+        intpart -= 1
+        fracpart += 1000000
+    assert 0 <= fracpart < 1000000
+    rffi.setintfield(l_timeval, 'c_tv_sec', intpart)
+    rffi.setintfield(l_timeval, 'c_tv_usec', fracpart)
 
 if not _WIN32:
     TMSP = lltype.Ptr(TMS)
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -59,6 +59,17 @@
         compile(f, (str, float))(str(fname), t1)
         assert t1 == os.stat(str(fname)).st_mtime
 
+    def test_utime_negative_fraction(self):
+        def f(fname, t1):
+            os.utime(fname, (t1, t1))
+
+        fname = udir.join('test_utime_negative_fraction.txt')
+        fname.ensure()
+        t1 = -123.75
+        compile(f, (str, float))(str(fname), t1)
+        got = os.stat(str(fname)).st_mtime
+        assert got == -123 or got == -123.75
+
     @win_only
     def test__getfullpathname(self):
         posix = __import__(os.name)

From pypy.commits at gmail.com  Tue Nov  8 07:25:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:25:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: no-op merge
Message-ID: <5821c42f.6737c20a.fe636.51bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88209:d955d28fddcd
Date: 2016-11-08 13:20 +0100
http://bitbucket.org/pypy/pypy/changeset/d955d28fddcd/

Log:	no-op merge


From pypy.commits at gmail.com  Tue Nov  8 07:25:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:25:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Un-Windows-ify line endings
Message-ID: <5821c431.cf3fc20a.58932.0098@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88210:96932525fa42
Date: 2016-11-08 13:24 +0100
http://bitbucket.org/pypy/pypy/changeset/96932525fa42/

Log:	Un-Windows-ify line endings

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,12 +1,12 @@
-==========================
-What's new in PyPy2.7 5.6+
-==========================
-
-.. this is a revision shortly after release-pypy2.7-v5.6
-.. startrev: 7e9787939641
-
-.. branch: rpython-error-to-systemerror
-
-Any uncaught RPython exception (from a PyPy bug) is turned into an
-app-level SystemError.  This should improve the lot of users hitting an
-uncaught RPython error.
+==========================
+What's new in PyPy2.7 5.6+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
+
+.. branch: rpython-error-to-systemerror
+
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.

From pypy.commits at gmail.com  Tue Nov  8 07:39:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:39:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Move the exception-catching scope to
 also include the call for generators
Message-ID: <5821c76b.d39a1c0a.435b2.9362@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88211:40d99b61ca87
Date: 2016-11-08 13:32 +0100
http://bitbucket.org/pypy/pypy/changeset/40d99b61ca87/

Log:	Move the exception-catching scope to also include the call for
	generators

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -277,13 +277,12 @@
                     next_instr = r_uint(self.last_instr + 1)
                     if next_instr != 0:
                         self.pushvalue(w_inputvalue)
-                try:
-                    w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                                executioncontext)
-                except OperationError:
-                    raise
-                except Exception as e:      # general fall-back
-                    raise self._convert_unexpected_exception(e)
+                w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                            executioncontext)
+            except OperationError:
+                raise
+            except Exception as e:      # general fall-back
+                raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None

From pypy.commits at gmail.com  Tue Nov  8 07:39:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:39:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5821c76e.c24bc20a.3a44a.9d29@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88212:84f687b458a4
Date: 2016-11-08 13:36 +0100
http://bitbucket.org/pypy/pypy/changeset/84f687b458a4/

Log:	hg merge default

diff too long, truncating to 2000 out of 11118 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -2,6 +2,7 @@
 from ctypes.test import need_symbol, xfail
 import unittest
 import os
+import sys
 
 import ctypes
 import _ctypes_test
@@ -279,7 +280,6 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
 
-    @xfail
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
         # Issue #23319
@@ -295,5 +295,11 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
 
+    # see issue #1213, on big endian it fails for the little endian case
+    if sys.byteorder == 'little':
+        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
+    elif sys.byteorder == 'big':
+        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -9,8 +9,8 @@
   Philip Jenvey
   Ronan Lamy
   Brian Kearns
+  Richard Plangger
   Michael Hudson
-  Richard Plangger
   Manuel Jacob
   David Schneider
   Holger Krekel
@@ -54,10 +54,10 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
-  Spenser Bauman
   Tyler Wade
   Vincent Legoll
   Michael Foord
@@ -65,13 +65,13 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Stefano Rivera
   Timo Paulssen
   Edd Barrett
   Squeaky
@@ -89,12 +89,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -128,10 +128,11 @@
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -147,17 +148,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
   Daniel Patrick
-  Jasper Schulz
-  Christian Hudon
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -165,8 +165,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -175,11 +175,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -196,7 +197,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -219,6 +219,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -231,6 +232,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -258,9 +260,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -274,10 +274,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
-  "Aaron Gallagher
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -287,8 +287,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -302,13 +303,11 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
-  JohnDoe
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
   reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
@@ -316,25 +315,24 @@
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
   remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
   werat
-  hgattic
+
+
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -1,4 +1,4 @@
-The PyPy Release Process
+PyPy's Release Process
 ========================
 
 Release Policy
@@ -39,41 +39,60 @@
   Some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
   necessary.
-* update pypy/doc/contributor.rst (and possibly LICENSE)
-  pypy/doc/tool/makecontributor.py generates the list of contributors
-* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  create a fresh whatsnew_head.rst after the release
-  and add the new file to  pypy/doc/index-of-whatsnew.rst
-* go to pypy/tool/release and run
-  ``force-builds.py <release branch>``
-  The following JIT binaries should be built, however, we need more buildbots
-  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
-  freebsd64 
 
-* wait for builds to complete, make sure there are no failures
-* download the builds, repackage binaries. Tag the release version
-  and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+* Update and write documentation
 
-  Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
-  and to cobra, as some packagers prefer a clearly labeled source package
-  ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
-  unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+  * update pypy/doc/contributor.rst (and possibly LICENSE)
+    pypy/doc/tool/makecontributor.py generates the list of contributors
 
-* Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+  * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+    create a fresh whatsnew_head.rst after the release
+    and add the new file to  pypy/doc/index-of-whatsnew.rst
 
-* write release announcement pypy/doc/release-x.y(.z).txt
+  * write release announcement pypy/doc/release-VERSION.rst
+    The release announcement should contain a direct link to the download page
 
-  The release announcement should contain a direct link to the download page
+  * Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
 
-* Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
+* Build and upload the release tar-balls
 
-* update pypy.org (under extradoc/pypy.org), rebuild and commit
+  * go to pypy/tool/release and run
+    ``force-builds.py <release branch>``
+    The following JIT binaries should be built, however, we need more buildbots
+    windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+    freebsd64 
 
-* post announcement on morepypy.blogspot.com
-* send announcements to twitter.com, pypy-dev, python-list,
-  python-announce, python-dev ...
+  * wait for builds to complete, make sure there are no failures
 
-* add a tag on the pypy/jitviewer repo that corresponds to pypy release
-* add a tag on the codespeed web site that corresponds to pypy release
-* revise versioning at https://readthedocs.org/projects/pypy
+  * send out a mailing list message asking for people to test before uploading
+    to prevent having to upload more than once
+
+  * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
+    that the source tarball can be produced in the next steps
+
+  * download the builds, repackage binaries. Tag the release version
+    and download and repackage source from bitbucket. You may find it
+    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+
+    Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
+    and to cobra, as some packagers prefer a clearly labeled source package
+    ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
+    unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
+  * Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+
+* Send out a mailing list message asking for last-minute comments and testing
+
+* RELEASE !  
+
+  * update pypy.org (under extradoc/pypy.org), rebuild and commit, using the
+    hashes produced from the ``repackage.sh`` script or by hand
+
+  * post announcement on morepypy.blogspot.com
+  * send announcements to twitter.com, pypy-dev, python-list,
+    python-announce, python-dev ...
+
+* If all is OK, document the released version
+
+  * add a tag on the codespeed web site that corresponds to pypy release
+  * revise versioning at https://readthedocs.org/projects/pypy
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -6,7 +6,7 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
 
 We continue to make incremental improvements to our C-API
-compatability layer (cpyext). We pass all but a few of the tests in the
+compatibility layer (cpyext). We pass all but a few of the tests in the
 upstream numpy `test suite`_. 
 
 Work proceeds at a good pace on the PyPy3.5
@@ -16,10 +16,11 @@
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
-We now support building PyPy with OpenSSL 1.1 in our built-in _sll module, as
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
 well as maintaining support for previous versions.
 
-XXX
+CFFI_ has been updated to 1.9, improving an already great package for
+interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -38,7 +39,7 @@
 with making RPython's JIT even better.
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
-.. _cffi: https://cffi.readthedocs.org
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
@@ -73,6 +74,7 @@
 =========================================================
 
 * New features
+
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
   * Backport fixes to rposix on windows from py3.5
@@ -81,20 +83,20 @@
   * Support more attributes on ``super``
   * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
   * Restore the ability to translate with CPython
-  * Update to CFFI 1.8.4
+  * Update to CFFI 1.9.0
   * Support the new buffer protocol in cpyext and numpypy
   * Add ``rposix.sync()``
   * Support full-precision nanosecond times in os.stat()
   * Add documentation about the assembler backends to RPYthon
   * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
-    changes downstream packaging requirments
+    changes downstream packaging requirements
   * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
   * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
     be ``str`` or ``unicode``)
   * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
   * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
     relevant platforms
-  * Backport the ``'faulthandler`` module from py3.5
+  * Backport the ``faulthandler`` module from py3.5
   * Improve the error message when trying to call a method where the ``self``
     parameter is missing in the definition
   * Implement ``rposix.cpu_count``
@@ -105,6 +107,7 @@
   * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
+
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
   * Fix translation of the sandbox
   * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
@@ -125,6 +128,7 @@
   * ``_numpypy.add.reduce`` returns a scalar now
 
 * Performance improvements:
+
   * Improve method calls on oldstyle classes
   * Clean and refactor code for testing cpyext to allow sharing with py3.5
   * Refactor a building the map of reflected ops in ``_numpypy``
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -111,4 +111,4 @@
         "hg log -r %s --template '{branch}'" % startrev)
     if errcode != 0:
         py.test.skip('no Mercurial repo')
-    assert wc_branch == 'default'
+    assert wc_branch in ('default', "'default'") # sometimes the ' leaks (windows)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -32,7 +32,7 @@
     'Niklaus Haldimann': ['nik'],
     'Alexander Schremmer': ['xoraxax'],
     'Anders Hammarquist': ['iko'],
-    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'David Edelsohn': ['edelsoh', 'edelsohn','opassembler.py'],
     'Niko Matsakis': ['niko'],
     'Jakub Gustak': ['jlg'],
     'Guido Wesdorp': ['guido'],
@@ -75,6 +75,9 @@
     'Spenser Bauman':['Spenser Andrew Bauman'],
     'Raffael Tfirst':['raffael.tfirst at gmail.com'],
     'timo':['timo at eistee.fritz.box'],
+    'Jasper Schulz':['Jasper.Schulz'],
+    'Aaron Gallagher':['"Aaron Gallagher'],
+    'Yasir Suhail':['yasirs'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,9 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.4+
+What's new in PyPy2.7 5.6+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 522736f816dc
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
 
 .. branch: mappingproxy
 .. branch: py3k-finish_time
@@ -14,92 +14,8 @@
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
-.. branch: buffer-interface
-Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
-in numpypy
+.. branch: rpython-error-to-systemerror
 
-.. branch: force-virtual-state
-Improve merging of virtual states in the JIT in order to avoid jumping to the
-preamble. Accomplished by allocating virtual objects where non-virtuals are
-expected.
-
-.. branch: conditional_call_value_3
-JIT residual calls: if the called function starts with a fast-path
-like "if x.foo != 0: return x.foo", then inline the check before
-doing the CALL.  For now, string hashing is about the only case.
-
-.. branch: search-path-from-libpypy
-
-The compiled pypy now looks for its lib-python/lib_pypy path starting
-from the location of the *libpypy-c* instead of the executable. This is
-arguably more consistent, and also it is what occurs anyway if you're
-embedding pypy.  Linux distribution packagers, take note!  At a minimum,
-the ``libpypy-c.so`` must really be inside the path containing
-``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
-from somewhere else.  You no longer have to do the same with the
-``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
-
-.. branch: _warnings
-
-CPython allows warning.warn(('something', 1), Warning), on PyPy this
-produced a "expected a readable buffer object" error. Test and fix.
-
-.. branch: stricter-strip
-
-CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
-allowed as arguments. Test and fix for str and unicode
-
-.. branch: faulthandler
-
-Port the 'faulthandler' module to PyPy default.  This module is standard
-in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
-
-.. branch: test-cpyext
-
-Refactor cpyext testing to be more pypy3-friendly.
-
-.. branch: better-error-missing-self
-
-Improve the error message when the user forgot the "self" argument of a method.
-
-
-.. fb6bb835369e
-Change the ``timeit`` module: it now prints the average time and the standard
-deviation over 7 runs by default, instead of the minimum. The minimum is often
-misleading.
-
-.. branch: unrecursive-opt
-
-Make optimiseopt iterative instead of recursive so it can be reasoned about
-more easily and debugging is faster.
-
-.. branch: Tiberiumk/fix-2412-1476011166874
-.. branch: redirect-assembler-jitlog
-
-
-
-.. branch: stdlib-2.7.12
-
-Update stdlib to version 2.7.12
-
-.. branch: buffer-interface2
-
-Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
-types still missing
-
-.. branch: fix-struct-unpack-Q
-
-Improve compatibility with CPython in the ``struct`` module. In particular,
-``struct.unpack`` now returns an ``int`` whenever the returned value fits,
-while previously it always returned a ``long`` for certains format codes such
-as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
-
-.. branch: newinitwarn
-
-Issue warnings for stricter handling of ``__new/init__`` args (that
-become TypeErrors in python 3)
-
-.. branch: openssl-1.1
-
-PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
-installed.  Thanks tumbleweed!
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.4+
-==========================
+=========================
+What's new in PyPy2.7 5.6
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
@@ -87,6 +87,7 @@
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
 
+
 .. branch: fix-struct-unpack-Q
 
 Improve compatibility with CPython in the ``struct`` module. In particular,
@@ -94,12 +95,19 @@
 while previously it always returned a ``long`` for certains format codes such
 as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
 
+.. branch: zarch-simd-support
+
+s390x implementation for vector operations used in VecOpt
+
+.. branch: ppc-vsx-support
+
+PowerPC implementation for vector operations used in VecOpt
+
 .. branch: newinitwarn
 
-Issue warnings for stricter handling of ``__new/init__`` args (that
-become TypeErrors in python 3)
+Match CPython's stricter handling of __new/init__ arguments
 
 .. branch: openssl-1.1
 
-PyPy can now be translated on a machine where the newer OpenSSL 1.1 is
-installed.  Thanks tumbleweed!
+Support for OpenSSL version 1.1 (in addition to version 1.0).
+Tested on Linux (1.1, 1.0), on Win32, and Mac (1.0 only)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -20,6 +20,22 @@
 
 .. _our downloads: http://pypy.org/download.html
 
+Installing Visual Compiler v9 (for Python 2.7)
+----------------------------------------------
+
+This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
+was checked in Nov 2016). Note that the compiler suite will be installed in
+``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
+Using a current version of ``setuptools`` will be able to find it there. For
+Windows 10, you must right-click the download, and under ``Properties`` ->
+``Compatibility`` mark it as ``Run run this program in comatibility mode for``
+``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
+otherwise ``mt.exe`` will silently fail. Installation will begin automatically
+by running the mt.exe command by hand from a DOS window (that is how the author
+discovered the problem).
+
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
 
 Translating PyPy with Visual Studio
 -----------------------------------
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,6 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib import rstackovf
 
 from pypy.interpreter import debug
 
@@ -65,10 +66,14 @@
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
         s = self._w_value
-        if self.__class__ is not OperationError and s is None:
-            space = getattr(self.w_type, 'space')
-            if space is not None:
+        space = getattr(self.w_type, 'space', None)
+        if space is not None:
+            if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
+            try:
+                s = space.str_w(s)
+            except Exception:
+                pass
         return '[%s: %s]' % (self.w_type, s)
 
     def __repr__(self):
@@ -632,3 +637,55 @@
 def raise_import_error(space, w_msg, w_name, w_path):
     w_exc = new_import_error(space, w_msg, w_name, w_path)
     raise OperationError(space.w_ImportError, w_exc)
+
+def _convert_unexpected_exception_extra(space, e):
+    "NOT_RPYTHON"
+    if e.__class__.__name__ in (
+        'Skipped',     # list of exception class names that are ok
+        ):             # to get during ==untranslated tests== only
+        raise
+    # include the RPython-level traceback
+    exc = sys.exc_info()
+    import traceback, cStringIO
+    f = cStringIO.StringIO()
+    print >> f, "\nTraceback (interpreter-level):"
+    traceback.print_tb(exc[2], file=f)
+    return f.getvalue()
+
+ at jit.dont_look_inside
+def get_converted_unexpected_exception(space, e):
+    """This is used in two places when we get an non-OperationError
+    RPython exception: from gateway.py when calling an interp-level
+    function raises; and from pyopcode.py when we're exiting the
+    interpretation of the frame with an exception.  Note that it
+    *cannot* be used in pyopcode.py: that place gets a
+    ContinueRunningNormally exception from the JIT, which must not end
+    up here!
+    """
+    try:
+        if not we_are_translated():
+            raise
+        raise e
+    except KeyboardInterrupt:
+        return OperationError(space.w_KeyboardInterrupt, space.w_None)
+    except MemoryError:
+        return OperationError(space.w_MemoryError, space.w_None)
+    except NotImplementedError:   # not on top of pypy! tests only
+        return OperationError(space.w_SystemError,
+                              space.wrap("NotImplementedError"))
+    except rstackovf.StackOverflow as e:
+        rstackovf.check_stack_overflow()
+        return oefmt(space.w_RecursionError,
+                     "maximum recursion depth exceeded")
+    except RuntimeError:   # not on top of py.py
+        return OperationError(space.w_RuntimeError, space.w_None)
+    except:
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
+        else:
+            extra = _convert_unexpected_exception_extra(space, e)
+        return OperationError(space.w_SystemError, space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -23,7 +23,6 @@
     DescrMismatch)
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode
-from rpython.rlib import rstackovf
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
@@ -756,16 +755,11 @@
             if not we_are_translated():
                 raise
             raise e
-        except KeyboardInterrupt:
-            raise OperationError(space.w_KeyboardInterrupt, space.w_None)
-        except MemoryError:
-            raise OperationError(space.w_MemoryError, space.w_None)
-        except rstackovf.StackOverflow as e:
-            rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RecursionError,
-                        "maximum recursion depth exceeded")
-        except RuntimeError:   # not on top of py.py
-            raise OperationError(space.w_RuntimeError, space.w_None)
+        except OperationError:
+            raise
+        except Exception as e:      # general fall-back
+            from pypy.interpreter import error
+            raise error.get_converted_unexpected_exception(space, e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,6 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+import sys
 from rpython.rlib import jit, rweakref
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
@@ -319,6 +320,12 @@
                 else:
                     w_exitvalue = self.dispatch(self.pycode, next_instr,
                                                 executioncontext)
+            except OperationError:
+                raise
+            except Exception as e:      # general fall-back
+                raise self._convert_unexpected_exception(e)
+                w_exitvalue = self.dispatch(self.pycode, next_instr,
+                                            executioncontext)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
@@ -958,6 +965,14 @@
             self.locals_cells_stack_w[i] = None
         self.valuestackdepth = 0
 
+    def _convert_unexpected_exception(self, e):
+        from pypy.interpreter import error
+
+        operr = error.get_converted_unexpected_exception(self.space, e)
+        pytraceback.record_application_traceback(
+            self.space, operr, self, self.last_instr)
+        raise operr
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -1,8 +1,10 @@
 import py
 from pypy.interpreter import executioncontext
+from pypy.interpreter.error import OperationError
 
-class Finished(Exception):
-    pass
+class Finished(OperationError):
+    def __init__(self):
+        OperationError.__init__(self, "exception_class", "exception_value")
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -4,6 +4,7 @@
 from pypy.interpreter import gateway, argument
 from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.error import OperationError
 import py
 import sys
 
@@ -789,6 +790,38 @@
         w_g = space.wrap(gateway.interp2app_temp(g, doc='bar'))
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
+    def test_system_error(self):
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        e = py.test.raises(OperationError, space.appexec, [w_g], """(my_g):
+            my_g()
+        """)
+        err = str(e.value)
+        assert 'SystemError' in err
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
+    def test_system_error_2(self):
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_msg = space.appexec([w_g], """(my_g):
+            try:
+                my_g()
+            except SystemError as e:
+                return str(e)
+        """)
+        err = space.str_w(w_msg)
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
     def test_unwrap_spec_default_bytes(self):
         space = self.space
         @gateway.unwrap_spec(s='bufferstr')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -76,7 +76,9 @@
         return space.w_None
     return space.get(w_descr, w_obj)
 
-def do_what_I_mean(space):
+def do_what_I_mean(space, w_crash=None):
+    if not space.is_none(w_crash):
+        raise ValueError    # RPython-level, uncaught
     return space.wrap(42)
 
 
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -73,6 +73,7 @@
         from __pypy__ import do_what_I_mean
         x = do_what_I_mean()
         assert x == 42
+        raises(SystemError, do_what_I_mean, 1)
 
     def test_list_strategy(self):
         from __pypy__ import strategy
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -325,6 +325,8 @@
             self.cdll = libffi.CDLL(name, mode)
         except DLOpenError as e:
             raise wrap_dlopenerror(space, e, self.name)
+        except OSError as e:
+            raise wrap_oserror(space, e)
 
     def getfunc(self, space, w_name, w_argtypes, w_restype):
         return _getfunc(space, self, w_name, w_argtypes, w_restype)
@@ -339,6 +341,9 @@
                         "No symbol %s found in library %s", name, self.name)
         return space.wrap(address_as_uint)
 
+    def getidentifier(self, space):
+        return space.wrap(self.cdll.getidentifier())
+
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
     return space.wrap(W_CDLL(space, name, mode))
@@ -349,6 +354,8 @@
     __new__     = interp2app(descr_new_cdll),
     getfunc     = interp2app(W_CDLL.getfunc),
     getaddressindll = interp2app(W_CDLL.getaddressindll),
+    __int__     = interp2app(W_CDLL.getidentifier),
+    __long__    = interp2app(W_CDLL.getidentifier),
     )
 
 class W_WinDLL(W_CDLL):
@@ -366,13 +373,11 @@
     __new__     = interp2app(descr_new_windll),
     getfunc     = interp2app(W_WinDLL.getfunc),
     getaddressindll = interp2app(W_WinDLL.getaddressindll),
+    __int__     = interp2app(W_CDLL.getidentifier),
+    __long__    = interp2app(W_CDLL.getidentifier),
     )
 
 # ========================================================================
 
 def get_libc(space):
-    try:
-        return space.wrap(W_CDLL(space, get_libc_name(), -1))
-    except OSError as e:
-        raise wrap_oserror(space, e)
-
+    return space.wrap(W_CDLL(space, get_libc_name(), -1))
diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py
--- a/pypy/module/_rawffi/alt/test/test_funcptr.py
+++ b/pypy/module/_rawffi/alt/test/test_funcptr.py
@@ -645,3 +645,21 @@
         f_name = libfoo.getfunc('AAA_first_ordinal_function', [], types.sint)
         f_ordinal = libfoo.getfunc(1, [], types.sint)
         assert f_name.getaddr() == f_ordinal.getaddr()
+
+    def test_cdll_as_integer(self):
+        import _rawffi
+        from _rawffi.alt import CDLL
+        libfoo = CDLL(self.libfoo_name)
+        A = _rawffi.Array('i')
+        a = A(1, autofree=True)
+        a[0] = libfoo      # should cast libfoo to int/long automatically
+
+    def test_windll_as_integer(self):
+        if not self.iswin32:
+            skip("windows specific")
+        import _rawffi
+        from _rawffi.alt import WinDLL
+        libm = WinDLL(self.libm_name)
+        A = _rawffi.Array('i')
+        a = A(1, autofree=True)
+        a[0] = libm        # should cast libm to int/long automatically
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -231,14 +231,17 @@
             raise oefmt(space.w_ValueError, "Cannot find symbol %s", name)
         return space.wrap(address_as_uint)
 
+def open_cdll(space, name):
+    try:
+        return CDLL(name)
+    except DLOpenError as e:
+        raise wrap_dlopenerror(space, e, name or "<None>")
+    except OSError as e:
+        raise wrap_oserror(space, e)
+
 @unwrap_spec(name='str_or_None')
 def descr_new_cdll(space, w_type, name):
-    try:
-        cdll = CDLL(name)
-    except DLOpenError as e:
-        raise wrap_dlopenerror(space, e, name)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    cdll = open_cdll(space, name)
     return space.wrap(W_CDLL(space, name, cdll))
 
 W_CDLL.typedef = TypeDef(
@@ -623,10 +626,7 @@
 
 def get_libc(space):
     name = get_libc_name()
-    try:
-        cdll = CDLL(name)
-    except OSError as e:
-        raise wrap_oserror(space, e)
+    cdll = open_cdll(space, name)
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,4 +1,3 @@
-
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,8 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.eval import (
     Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
+from pypy.module.cpyext.api import (c_fopen, c_fclose, c_fileno, 
+                Py_ssize_tP, is_valid_fd)
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.astcompiler import consts
 from rpython.tool.udir import udir
@@ -142,7 +143,8 @@
         fp = c_fopen(str(filepath), "rb")
         os.close(c_fileno(fp))
         api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
-        c_fclose(fp)
+        if is_valid_fd(c_fileno(fp)):
+            c_fclose(fp)
         assert api.PyErr_Occurred() is space.w_IOError
         api.PyErr_Clear()
 
diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -1,4 +1,3 @@
-
 class AppTestFaultHandler:
     spaceconfig = {
         "usemodules": ["faulthandler", "_vmprof"]
diff --git a/pypy/module/gc/test/test_app_referents.py b/pypy/module/gc/test/test_app_referents.py
--- a/pypy/module/gc/test/test_app_referents.py
+++ b/pypy/module/gc/test/test_app_referents.py
@@ -4,12 +4,13 @@
 
 def test_interface_to_dump_rpy_heap_str(space):
     filename = str(udir.join('dump_rpy_heap.str'))
-    try:
-        space.appexec([space.wrap(filename)], """(filename):
-            import gc
-            gc.dump_rpy_heap(filename)""")
-    except NotImplementedError:
-        pass
+    space.appexec([space.wrap(filename)], """(filename):
+        import gc
+        try:
+            gc.dump_rpy_heap(filename)
+        except SystemError:
+            pass
+    """)
     assert os.path.exists(filename)
 
 def test_interface_to_dump_rpy_heap_file(space):
@@ -20,20 +21,22 @@
             f.write(b'X')
             return f""")
     assert os.path.getsize(filename) == 0   # the 'X' was not flushed yet
-    try:
-        space.appexec([w_f], """(f):
-            import gc
-            gc.dump_rpy_heap(f)""")
-    except NotImplementedError:
-        pass
+    space.appexec([w_f], """(f):
+        import gc
+        try:
+            gc.dump_rpy_heap(f)
+        except SystemError:
+            pass
+    """)
     assert os.path.getsize(filename) == 1   # the 'X' was flushed here
 
 def test_interface_to_dump_rpy_heap_fd(space):
     filename = str(udir.join('dump_rpy_heap.fd'))
     f = open(filename, 'wb')
-    try:
-        space.appexec([space.wrap(f.fileno())], """(fd):
-            import gc
-            gc.dump_rpy_heap(fd)""")
-    except NotImplementedError:
-        pass
+    space.appexec([space.wrap(f.fileno())], """(fd):
+        import gc
+        try:
+            gc.dump_rpy_heap(fd)
+        except SystemError:
+            pass
+    """)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -16,7 +16,7 @@
 from rpython.jit.backend.detect_cpu import getcpuclass
 
 CPU = getcpuclass()
-if not CPU.vector_extension:
+if not CPU.vector_ext:
     py.test.skip("this cpu %s has no implemented vector backend" % CPU)
 
 def get_profiler():
@@ -29,7 +29,7 @@
     interp = None
 
     def setup_method(self, method):
-        if not self.CPUClass.vector_extension:
+        if not self.CPUClass.vector_ext:
             py.test.skip("needs vector extension to run (for now)")
 
     def assert_float_equal(self, f1, f2, delta=0.0001):
@@ -418,9 +418,7 @@
     def test_sum_float_to_int16(self):
         result = self.run("sum_float_to_int16")
         assert result == sum(range(30))
-        # one can argue that this is not desired,
-        # but unpacking exactly hits savings = 0
-        self.check_vectorized(1, 1)
+
     def define_sum_float_to_int32():
         return """
         a = |30|
@@ -429,7 +427,6 @@
     def test_sum_float_to_int32(self):
         result = self.run("sum_float_to_int32")
         assert result == sum(range(30))
-        self.check_vectorized(1, 1)
 
     def define_sum_float_to_float32():
         return """
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1009,7 +1009,8 @@
             pid1, status1 = os.waitpid(pid, 0)
             assert pid1 == pid
             assert os.WIFEXITED(status1)
-            assert os.WEXITSTATUS(status1) == myprio + 3
+            expected = min(myprio + 3, 19)
+            assert os.WEXITSTATUS(status1) == expected
 
     if hasattr(os, 'symlink'):
         def test_symlink(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -418,7 +418,7 @@
         guard_no_exception(descr=...)
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(ptr85), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>)
-        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_length .+>)
+        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_allocated_length .+>)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -3,23 +3,35 @@
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 from rpython.rlib.rawstorage import misaligned_is_fine
 
+def no_vector_backend():
+    import platform
+    if platform.machine().startswith('x86'):
+        from rpython.jit.backend.x86.detect_feature import detect_sse4_2
+        return not detect_sse4_2()
+    if platform.machine().startswith('ppc'):
+        from rpython.jit.backend.ppc.detect_feature import detect_vsx
+        return not detect_vsx()
+    if platform.machine() == "s390x":
+        from rpython.jit.backend.zarch.detect_feature import detect_simd_z
+        return not detect_simd_z()
+    return True
 
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
-             ('+','float','int',   9*7843,   7843, 4.0,5.0),
-             ('+','int','float',   8*2571,   2571, 9.0,-1.0),
-             ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
-             ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
-             ('-','float','float', -2*5523,  5523, 1.0,3.0),
-             ('*','float','float', 3*2999,   2999, 1.0,3.0),
-             ('/','float','float', 3*7632,   7632, 3.0,1.0),
-             ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
-             ('&','int','int',     0,        1500, 1,0),
-             ('&','int','int',     1500,     1500, 1,1),
-             ('|','int','int',     1500,     1500, 0,1),
-             ('|','int','int',     0,        1500, 0,0),
-            ]
+                  ('+','float','int',   9*7843,   7843, 4.0,5.0),
+                  ('+','int','float',   8*2571,   2571, 9.0,-1.0),
+                  ('+','float','int',   -18*2653,   2653, 4.0,-22.0),
+                  ('+','int','int',     -1*1499,   1499, 24.0,-25.0),
+                  ('-','float','float', -2*5523,  5523, 1.0,3.0),
+                  ('*','float','float', 3*2999,   2999, 1.0,3.0),
+                  ('/','float','float', 3*7632,   7632, 3.0,1.0),
+                  ('/','float','float', 1.5*7632, 7632, 3.0,2.0),
+                  ('&','int','int',     0,        1500, 1,0),
+                  ('&','int','int',     1500,     1500, 1,1),
+                  ('|','int','int',     1500,     1500, 0,1),
+                  ('|','int','int',     0,        1500, 0,0),
+                 ]
     type_permuated = []
     types = { 'int': ['int32','int64','int8','int16'],
               'float': ['float32', 'float64']
@@ -36,6 +48,7 @@
                 type_permuated.append(t)
 
     @py.test.mark.parametrize("op,adtype,bdtype,result,count,a,b", type_permuated)
+    @py.test.mark.skipif('no_vector_backend()')
     def test_vector_call2(self, op, adtype, bdtype, result, count, a, b):
         source = """
         def main():
@@ -51,6 +64,13 @@
         log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        if adtype in ('int64','float64') and bdtype in ('int64','float64'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
 
     arith_comb = [
@@ -58,7 +78,7 @@
         ('sum','float', 2581, 2581, 1),
         ('prod','float', 1, 3178, 1),
         ('prod','int', 1, 3178, 1),
-        ('any','int', 1, 1239, 1),
+        ('any','int', 1, 2239, 1),
         ('any','int', 0, 4912, 0),
         ('all','int', 0, 3420, 0),
         ('all','int', 1, 6757, 1),
@@ -76,6 +96,7 @@
             type_permuated.append(t)
 
     @py.test.mark.parametrize("op,dtype,result,count,a", type_permuated)
+    @py.test.mark.skipif('no_vector_backend()')
     def test_reduce_generic(self,op,dtype,result,count,a):
         source = """
         def main():
@@ -84,10 +105,19 @@
             return a.{method}()
         """.format(method=op, dtype=dtype, count=count, a=a)
         exec py.code.Source(source).compile()
+        log = self.run(main, [], vec=0)
         vlog = self.run(main, [], vec=1)
-        log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
+        if not log.jit_summary:
+            return
+        assert log.jit_summary.vecopt_tried == 0
+        assert log.jit_summary.vecopt_success == 0
+        assert vlog.jit_summary.vecopt_tried > 0
+        if dtype in ('int64','float64') and (dtype != 'int64' and op != 'prod'):
+            assert vlog.jit_summary.vecopt_success > 0
+        else:
+            assert vlog.jit_summary.vecopt_success >= 0
 
     def test_reduce_logical_xor(self):
         def main():
@@ -158,7 +188,7 @@
         assert log.result is True
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
-        assert loop.match("""
+        loop.match("""
             f31 = raw_load_f(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
             i32 = float_ne(f31, 0.000000)
@@ -168,7 +198,20 @@
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
             jump(..., descr=...)
-        """)
+            """)
+        # vector version
+        #assert loop.match("""
+        #    guard_not_invalidated(descr=...)
+        #    i38 = int_add(i25, 2)
+        #    i39 = int_ge(i38, i33)
+        #    guard_false(i39, descr=...)
+        #    v42 = vec_load_f(i9, i32, 1, 0, descr=<ArrayF 8>)
+        #    v43 = vec_float_ne(v42, v36)
+        #    f46 = vec_unpack_f(v42, 0, 1)
+        #    vec_guard_true(v43, descr=...)
+        #    i48 = int_add(i32, 16)
+        #    i50 = int_add(i25, 2)
+        #    jump(..., descr=...)""")
 
     def test_array_getitem_basic(self):
         def main():
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -179,11 +179,19 @@
 @jit.dont_look_inside
 @unwrap_spec(timeout=int)
 def alarm(space, timeout):
+    """alarm(seconds)
+
+    Arrange for SIGALRM to arrive after the given number of seconds.
+    """
     return space.wrap(c_alarm(timeout))
 
 
 @jit.dont_look_inside
 def pause(space):
+    """pause()
+
+    Wait until a signal arrives.
+    """
     c_pause()
     return space.w_None
 
@@ -258,6 +266,12 @@
 @jit.dont_look_inside
 @unwrap_spec(signum=int, flag=int)
 def siginterrupt(space, signum, flag):
+    """siginterrupt(sig, flag) -> None
+
+    change system call restart behaviour: if flag is False, system calls
+    will be restarted when interrupted by signal sig, else system calls
+    will be interrupted.
+    """
     check_signum_in_range(space, signum)
     if rffi.cast(lltype.Signed, c_siginterrupt(signum, flag)) < 0:
         raise exception_from_saved_errno(space, space.w_OSError)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -829,65 +829,83 @@
     new_obj = map.materialize_r_dict(space, obj, dict_w)
     obj._set_mapdict_storage_and_map(new_obj.storage, new_obj.map)
 
+
+class IteratorMixin(object):
+
+    def _init(self, strategy, w_dict):
+        w_obj = strategy.unerase(w_dict.dstorage)
+        self.w_obj = w_obj
+        self.orig_map = curr_map = w_obj._get_mapdict_map()
+        # We enumerate non-lazily the attributes, and store them in the
+        # 'attrs' list.  We then walk that list in opposite order.  This
+        # gives an ordering that is more natural (roughly corresponding
+        # to oldest-first) than the one we get in the direct algorithm
+        # (from leaves to root, looks like backward order).  See issue
+        # #2426: it should improve the performance of code like
+        # copy.copy().
+        attrs = []
+        while True:
+            curr_map = curr_map.search(DICT)
+            if curr_map is None:
+                break
+            attrs.append(curr_map.name)
+            curr_map = curr_map.back
+        self.attrs = attrs
+
+
 class MapDictIteratorKeys(BaseKeyIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseKeyIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_key_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                w_attr = self.space.wrap(attr.decode('utf-8'))
-                return w_attr
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            w_attr = self.space.wrap(attr.decode('utf-8'))
+            return w_attr
         return None
 
 
 class MapDictIteratorValues(BaseValueIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseValueIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_value_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                return self.w_obj.getdictvalue(self.space, attr)
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            return self.w_obj.getdictvalue(self.space, attr)
         return None
 
 
 class MapDictIteratorItems(BaseItemIterator):
+    objectmodel.import_from_mixin(IteratorMixin)
+
     def __init__(self, space, strategy, w_dict):
         BaseItemIterator.__init__(self, space, strategy, w_dict)
-        w_obj = strategy.unerase(w_dict.dstorage)
-        self.w_obj = w_obj
-        self.orig_map = self.curr_map = w_obj._get_mapdict_map()
+        self._init(strategy, w_dict)
 
     def next_item_entry(self):
         assert isinstance(self.w_dict.get_strategy(), MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None, None
-        if self.curr_map:
-            curr_map = self.curr_map.search(DICT)
-            if curr_map:
-                self.curr_map = curr_map.back
-                attr = curr_map.name
-                w_attr = self.space.wrap(attr.decode('utf-8'))
-                return w_attr, self.w_obj.getdictvalue(self.space, attr)
+        attrs = self.attrs
+        if len(attrs) > 0:
+            attr = attrs.pop()
+            w_attr = self.space.wrap(attr.decode('utf-8'))
+            return w_attr, self.w_obj.getdictvalue(self.space, attr)
         return None, None
 
 
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1204,6 +1204,20 @@
         got = a.method()
         assert got == 43
 
+    def test_dict_order(self):
+        # the __dict__ order is not strictly enforced, but in
+        # simple cases like that, we want to follow the order of
+        # creation of the attributes
+        class A(object):
+            pass
+        a = A()
+        a.x = 5
+        a.z = 6
+        a.y = 7
+        assert list(a.__dict__) == ['x', 'z', 'y']
+        assert a.__dict__.values() == [5, 6, 7]
+        assert list(a.__dict__.iteritems()) == [('x', 5), ('z', 6), ('y', 7)]
+
     def test_bug_method_change(self):
         class A(object):
             def method(self):
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -194,6 +194,13 @@
 
 def _build(cfilenames, outputfilename, compile_extra, link_extra,
         include_dirs, libraries, library_dirs):
+    try:
+        # monkeypatch distutils for some versions of msvc compiler
+        import setuptools
+    except ImportError:
+        # XXX if this fails and is required, 
+        #     we must call pypy -mensurepip after translation
+        pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
     compiler = new_compiler(force=1)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -146,6 +146,27 @@
         # result should be an integer
         assert s.knowntype == int
 
+    def test_not_rpython(self):
+        def g(x):
+            """ NOT_RPYTHON """
+            return eval(x)
+        def f(x):
+            return g(str(x))
+        a = self.RPythonAnnotator()
+        with py.test.raises(ValueError):
+            a.build_types(f, [int])
+
+    def test_not_rpython_decorator(self):
+        from rpython.rlib.objectmodel import not_rpython
+        @not_rpython
+        def g(x):
+            return eval(x)
+        def f(x):
+            return g(str(x))
+        a = self.RPythonAnnotator()
+        with py.test.raises(ValueError):
+            a.build_types(f, [int])
+
     def test_lists(self):
         a = self.RPythonAnnotator()
         end_cell = a.build_types(snippet.poor_man_rev_range, [int])
diff --git a/rpython/doc/faq.rst b/rpython/doc/faq.rst
--- a/rpython/doc/faq.rst
+++ b/rpython/doc/faq.rst
@@ -80,6 +80,9 @@
 stops and reports this as an error. You can therefore mark functions as
 "NOT_RPYTHON" to make sure that they are never analyzed.
 
+This method of marking a function as not RPython is outdated. For new code,
+please use the ``@objectmodel.not_rpython`` decorator instead.
+
 
 Couldn't we simply take a Python syntax tree and turn it into Lisp?
 -------------------------------------------------------------------
@@ -143,7 +146,7 @@
   attempt to point newcomers at existing alternatives, which are more
   mainstream and where they will get help from many people.*
 
-  *If anybody seriously wants to promote RPython anyway, he is welcome
+  *If anybody seriously wants to promote RPython anyway, they are welcome
   to: we won't actively resist such a plan.  There are a lot of things
   that could be done to make RPython a better Java-ish language for
   example, starting with supporting non-GIL-based multithreading, but we
@@ -182,8 +185,8 @@
 patch the generated machine code.
 
 So the position of the core PyPy developers is that if anyone wants to
-make an N+1'th attempt with LLVM, he is welcome, and he will receive a
-bit of help on the IRC channel, but he is left with the burden of proof
+make an N+1'th attempt with LLVM, they are welcome, and they will receive a
+bit of help on the IRC channel, but they are left with the burden of proof
 that it works.
 
 
diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -13,11 +13,6 @@
   (e.g. those in the NumPyPy module).
 * --jit vec_all=1: turns on the vectorization for any jit driver. See parameters for
   the filtering heuristics of traces.
-* --jit vec_ratio=2: A number from 0 to 10 that represents a real number (vec_ratio / 10).
-  This filters traces if vec_all is enabled. N is the trace count then the number of
-  vector transformable operations (add_int -> vec_add_int) M, the following must hold:
-  M / N >= (vec_ratio / 10)
-* --jit vec_length=60: The maximum number of trace instructions the vectorizer filters for.
 
 Features
 --------
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -18,6 +18,8 @@
     except AttributeError:
         raise ValueError("%r is not RPython: it is likely an unexpected "
                          "built-in function or type" % (func,))
+    if getattr(func, "_not_rpython_", False):
+        raise ValueError("%r is tagged as @not_rpython" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -13,7 +13,6 @@
 MODEL_X86         = 'x86'
 MODEL_X86_NO_SSE2 = 'x86-without-sse2'
 MODEL_X86_64      = 'x86-64'
-MODEL_X86_64_SSE4 = 'x86-64-sse4'
 MODEL_ARM         = 'arm'
 MODEL_PPC_64      = 'ppc-64'
 MODEL_S390_64     = 's390x'
@@ -81,9 +80,6 @@
         from rpython.jit.backend.x86 import detect_feature as feature
         if sys.maxint == 2**63-1:
             result = MODEL_X86_64
-            # has sse 2 at least
-            if feature.detect_sse4_1():
-                result = MODEL_X86_64_SSE4
         else:
             assert sys.maxint == 2**31-1
             if feature.detect_sse2():
@@ -120,8 +116,6 @@
         return "rpython.jit.backend.x86.runner", "CPU386_NO_SSE2"
     elif backend_name == MODEL_X86_64:
         return "rpython.jit.backend.x86.runner", "CPU_X86_64"
-    elif backend_name == MODEL_X86_64_SSE4:
-        return "rpython.jit.backend.x86.runner", "CPU_X86_64_SSE4"
     elif backend_name == MODEL_ARM:
         return "rpython.jit.backend.arm.runner", "CPU_ARM"
     elif backend_name == MODEL_PPC_64:
@@ -145,7 +139,6 @@
         MODEL_X86: ['floats', 'singlefloats', 'longlong'],
         MODEL_X86_NO_SSE2: ['longlong'],
         MODEL_X86_64: ['floats', 'singlefloats'],
-        MODEL_X86_64_SSE4: ['floats', 'singlefloats'],
         MODEL_ARM: ['floats', 'singlefloats', 'longlong'],
         MODEL_PPC_64: ['floats'],
         MODEL_S390_64: ['floats'],
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend import model
 from rpython.jit.backend.llgraph import support
 from rpython.jit.backend.llsupport import symbolic
+from rpython.jit.backend.llsupport.vector_ext import VectorExt
 from rpython.jit.metainterp.history import AbstractDescr
 from rpython.jit.metainterp.history import Const, getkind
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID
@@ -328,10 +329,11 @@
     supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
-    vector_extension = True
-    vector_register_size = 16 # in bytes
-    vector_horizontal_operations = True
-    vector_pack_slots = True
+    vector_ext = VectorExt()
+    vector_ext.enable(16, accum=True)
+    vector_ext.setup_once = lambda asm: asm
+    load_supported_factors = (1,2,4,8)
+    assembler = None
 
     def __init__(self, rtyper, stats=None, *ignored_args, **kwds):
         model.AbstractCPU.__init__(self)
@@ -877,6 +879,9 @@
     def bh_vec_int_xor(self, vx, vy, count):
         return [int(x) ^ int(y) for x,y in zip(vx,vy)]
 
+    def bh_vec_float_xor(self, vx, vy, count):
+        return [0.0 for x,y in zip(vx,vy)] # just used for clearing the vector register
+
     def bh_vec_cast_float_to_singlefloat(self, vx, count):
         from rpython.rlib.rarithmetic import r_singlefloat
         return [longlong.singlefloat2int(r_singlefloat(longlong.getrealfloat(v)))
@@ -928,50 +933,32 @@
     def bh_vec_int_signext(self, vx, ext, count):
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
-    def build_getarrayitem(func):
-        def method(self, struct, offset, descr, _count):
+    def build_load(func):
+        def method(self, struct, offset, scale, disp, descr, _count):
             values = []
-            count = self.vector_register_size // descr.get_item_size_in_bytes()
+            count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
+            adr = struct + (offset * scale + disp)
+            a = support.cast_arg(lltype.Ptr(descr.A), adr)
+            array = a._obj
             for i in range(count):
-                val = func(self, struct, offset + i, descr)
+                val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
         return method
 
-    bh_vec_getarrayitem_gc_i = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_gc_f = build_getarrayitem(bh_getarrayitem_gc)
-    bh_vec_getarrayitem_raw_i = build_getarrayitem(bh_getarrayitem_raw)
-    bh_vec_getarrayitem_raw_f = build_getarrayitem(bh_getarrayitem_raw)
-    del build_getarrayitem
+    bh_vec_load_i = build_load(bh_getarrayitem_raw)
+    bh_vec_load_f = build_load(bh_getarrayitem_raw)
+    del build_load
 
-    def _bh_vec_raw_load(self, struct, offset, descr, _count):
-        values = []
+    def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
         stride = descr.get_item_size_in_bytes()
-        count = self.vector_register_size // descr.get_item_size_in_bytes()
-        assert _count == count
-        assert count > 0
-        for i in range(count):
-            val = self.bh_raw_load(struct, offset + i*stride, descr)
-            values.append(val)
-        return values
-
-    bh_vec_raw_load_i = _bh_vec_raw_load
-    bh_vec_raw_load_f = _bh_vec_raw_load
-
-    def bh_vec_raw_store(self, struct, offset, newvalues, descr, count):
-        stride = descr.get_item_size_in_bytes()
+        adr = struct + (offset * scale + disp)
+        a = support.cast_arg(lltype.Ptr(descr.A), adr)
+        array = a._obj
         for i,n in enumerate(newvalues):
-            self.bh_raw_store(struct, offset + i*stride, n, descr)
-
-    def bh_vec_setarrayitem_raw(self, struct, offset, newvalues, descr, count):

From pypy.commits at gmail.com  Tue Nov  8 07:39:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:39:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5821c770.4d881c0a.d56ba.ebba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88213:b60d335832b3
Date: 2016-11-08 13:37 +0100
http://bitbucket.org/pypy/pypy/changeset/b60d335832b3/

Log:	hg merge default


From pypy.commits at gmail.com  Tue Nov  8 07:39:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:39:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5821c772.8a0d1c0a.df97e.e483@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88214:0496d602181c
Date: 2016-11-08 13:38 +0100
http://bitbucket.org/pypy/pypy/changeset/0496d602181c/

Log:	fix test

diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1215,8 +1215,8 @@
         a.z = 6
         a.y = 7
         assert list(a.__dict__) == ['x', 'z', 'y']
-        assert a.__dict__.values() == [5, 6, 7]
-        assert list(a.__dict__.iteritems()) == [('x', 5), ('z', 6), ('y', 7)]
+        assert list(a.__dict__.values()) == [5, 6, 7]
+        assert list(a.__dict__.items()) == [('x', 5), ('z', 6), ('y', 7)]
 
     def test_bug_method_change(self):
         class A(object):

From pypy.commits at gmail.com  Tue Nov  8 07:58:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:58:54 -0800 (PST)
Subject: [pypy-commit] pypy default: Backed out changeset 4c405c663a1c
Message-ID: <5821cc0e.46bb1c0a.33fda.0f40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88215:b49ff4761bde
Date: 2016-11-08 13:52 +0100
http://bitbucket.org/pypy/pypy/changeset/b49ff4761bde/

Log:	Backed out changeset 4c405c663a1c

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -543,9 +543,6 @@
         return OperationError(space.w_KeyboardInterrupt, space.w_None)
     except MemoryError:
         return OperationError(space.w_MemoryError, space.w_None)
-    except NotImplementedError:   # not on top of pypy! tests only
-        return OperationError(space.w_SystemError,
-                              space.wrap("NotImplementedError"))
     except rstackovf.StackOverflow as e:
         rstackovf.check_stack_overflow()
         return oefmt(space.w_RuntimeError,
diff --git a/pypy/module/gc/test/test_app_referents.py b/pypy/module/gc/test/test_app_referents.py
--- a/pypy/module/gc/test/test_app_referents.py
+++ b/pypy/module/gc/test/test_app_referents.py
@@ -4,13 +4,12 @@
 
 def test_interface_to_dump_rpy_heap_str(space):
     filename = str(udir.join('dump_rpy_heap.str'))
-    space.appexec([space.wrap(filename)], """(filename):
-        import gc
-        try:
-            gc.dump_rpy_heap(filename)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([space.wrap(filename)], """(filename):
+            import gc
+            gc.dump_rpy_heap(filename)""")
+    except NotImplementedError:
+        pass
     assert os.path.exists(filename)
 
 def test_interface_to_dump_rpy_heap_file(space):
@@ -21,22 +20,20 @@
             f.write('X')
             return f""")
     assert os.path.getsize(filename) == 0   # the 'X' was not flushed yet
-    space.appexec([w_f], """(f):
-        import gc
-        try:
-            gc.dump_rpy_heap(f)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([w_f], """(f):
+            import gc
+            gc.dump_rpy_heap(f)""")
+    except NotImplementedError:
+        pass
     assert os.path.getsize(filename) == 1   # the 'X' was flushed here
 
 def test_interface_to_dump_rpy_heap_fd(space):
     filename = str(udir.join('dump_rpy_heap.fd'))
     f = open(filename, 'wb')
-    space.appexec([space.wrap(f.fileno())], """(fd):
-        import gc
-        try:
-            gc.dump_rpy_heap(fd)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([space.wrap(f.fileno())], """(fd):
+            import gc
+            gc.dump_rpy_heap(fd)""")
+    except NotImplementedError:
+        pass

From pypy.commits at gmail.com  Tue Nov  8 07:58:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 04:58:56 -0800 (PST)
Subject: [pypy-commit] pypy default: comment
Message-ID: <5821cc10.e128c20a.60b07.ac02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88216:6eccb6e5fe92
Date: 2016-11-08 13:58 +0100
http://bitbucket.org/pypy/pypy/changeset/6eccb6e5fe92/

Log:	comment

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -544,6 +544,13 @@
     except MemoryError:
         return OperationError(space.w_MemoryError, space.w_None)
     except rstackovf.StackOverflow as e:
+        # xxx twisted logic which happens to give the result that we
+        # want: when untranslated, a RuntimeError or its subclass
+        # NotImplementedError is caught here.  Then
+        # check_stack_overflow() will re-raise it directly.  We see
+        # the result as this exception propagates directly.  But when
+        # translated, an RPython-level RuntimeError is turned into
+        # an app-level RuntimeError by the next case.
         rstackovf.check_stack_overflow()
         return oefmt(space.w_RuntimeError,
                      "maximum recursion depth exceeded")

From pypy.commits at gmail.com  Tue Nov  8 08:12:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 05:12:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5821cf28.83f8c20a.ac90e.a8b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88217:17b695310d4d
Date: 2016-11-08 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/17b695310d4d/

Log:	hg merge default

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -670,10 +670,14 @@
         return OperationError(space.w_KeyboardInterrupt, space.w_None)
     except MemoryError:
         return OperationError(space.w_MemoryError, space.w_None)
-    except NotImplementedError:   # not on top of pypy! tests only
-        return OperationError(space.w_SystemError,
-                              space.wrap("NotImplementedError"))
     except rstackovf.StackOverflow as e:
+        # xxx twisted logic which happens to give the result that we
+        # want: when untranslated, a RuntimeError or its subclass
+        # NotImplementedError is caught here.  Then
+        # check_stack_overflow() will re-raise it directly.  We see
+        # the result as this exception propagates directly.  But when
+        # translated, an RPython-level RuntimeError is turned into
+        # an app-level RuntimeError by the next case.
         rstackovf.check_stack_overflow()
         return oefmt(space.w_RecursionError,
                      "maximum recursion depth exceeded")
diff --git a/pypy/module/gc/test/test_app_referents.py b/pypy/module/gc/test/test_app_referents.py
--- a/pypy/module/gc/test/test_app_referents.py
+++ b/pypy/module/gc/test/test_app_referents.py
@@ -4,13 +4,12 @@
 
 def test_interface_to_dump_rpy_heap_str(space):
     filename = str(udir.join('dump_rpy_heap.str'))
-    space.appexec([space.wrap(filename)], """(filename):
-        import gc
-        try:
-            gc.dump_rpy_heap(filename)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([space.wrap(filename)], """(filename):
+            import gc
+            gc.dump_rpy_heap(filename)""")
+    except NotImplementedError:
+        pass
     assert os.path.exists(filename)
 
 def test_interface_to_dump_rpy_heap_file(space):
@@ -21,22 +20,20 @@
             f.write(b'X')
             return f""")
     assert os.path.getsize(filename) == 0   # the 'X' was not flushed yet
-    space.appexec([w_f], """(f):
-        import gc
-        try:
-            gc.dump_rpy_heap(f)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([w_f], """(f):
+            import gc
+            gc.dump_rpy_heap(f)""")
+    except NotImplementedError:
+        pass
     assert os.path.getsize(filename) == 1   # the 'X' was flushed here
 
 def test_interface_to_dump_rpy_heap_fd(space):
     filename = str(udir.join('dump_rpy_heap.fd'))
     f = open(filename, 'wb')
-    space.appexec([space.wrap(f.fileno())], """(fd):
-        import gc
-        try:
-            gc.dump_rpy_heap(fd)
-        except SystemError:
-            pass
-    """)
+    try:
+        space.appexec([space.wrap(f.fileno())], """(fd):
+            import gc
+            gc.dump_rpy_heap(fd)""")
+    except NotImplementedError:
+        pass

From pypy.commits at gmail.com  Tue Nov  8 08:12:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 05:12:20 -0800 (PST)
Subject: [pypy-commit] pypy default: add intmask to arith in llgraph runner
 (was looking at the x86 backend all the time)
Message-ID: <5821cf34.c5311c0a.30fa2.1352@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88218:b1e0b4162789
Date: 2016-11-08 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/b1e0b4162789/

Log:	add intmask to arith in llgraph runner (was looking at the x86
	backend all the time)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -18,7 +18,7 @@
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
-from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
 from rpython.rlib.objectmodel import Symbolic
 
 class LLAsmInfo(object):
@@ -840,7 +840,7 @@
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
         assert len(vx) == len(vy) == count
-        return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
+        return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
     """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
     exec py.code.Source(vector_arith_code.format('int','sub','-')).compile()
@@ -891,9 +891,6 @@
         return [longlong.getfloatstorage(float(longlong.int2singlefloat(v)))
                 for v in vx]
 
-        a = float(a)
-        return longlong.getfloatstorage(a)
-
     def bh_vec_cast_float_to_int(self, vx, count):
         return [int(x) for x in vx]
 

From pypy.commits at gmail.com  Tue Nov  8 10:27:02 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 08 Nov 2016 07:27:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: make PyErr_SetFromErrnoWithFilename use
 wrap_fsdecoded
Message-ID: <5821eec6.542e1c0a.81dc1.8423@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88219:88143396c66f
Date: 2016-11-08 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/88143396c66f/

Log:	make PyErr_SetFromErrnoWithFilename use wrap_fsdecoded

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -147,7 +147,7 @@
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     if llfilename:
         filename = rffi.charp2str(llfilename)
-        w_filename = space.wrap(filename)
+        w_filename = space.wrap_fsdecoded(filename)
     else:
         w_filename = space.w_None
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -105,6 +105,13 @@
         assert api.PyOS_InterruptOccurred()
 
 class AppTestFetch(AppTestCpythonExtensionBase):
+    def setup_class(cls):
+        from pypy.module.imp.test.support import get_special_char
+        space = cls.space
+        cls.special_char = get_special_char()
+        cls.w_special_char = space.wrap(cls.special_char)
+        AppTestCpythonExtensionBase.setup_class.im_func(cls)
+
 
     def test_occurred(self):
         module = self.import_extension('foo', [
@@ -199,6 +206,9 @@
             assert e.filename is None
 
     def test_SetFromErrnoWithFilename(self):
+        char = self.special_char
+        if char is None:
+            char = "a" # boring
         import errno, os
 
         module = self.import_extension('foo', [
@@ -208,6 +218,12 @@
                  PyErr_SetFromErrnoWithFilename(PyExc_OSError, "/path/to/file");
                  return NULL;
                  '''),
+                ("set_from_errno_special", "METH_NOARGS",
+                 '''
+                 errno = EBADF;
+                 PyErr_SetFromErrnoWithFilename(PyExc_OSError, "/path/to/%s");
+                 return NULL;
+                 ''' % (char, )),
                 ],
                 prologue="#include <errno.h>")
         exc_info = raises(OSError, module.set_from_errno)
@@ -215,6 +231,11 @@
         assert exc_info.value.errno == errno.EBADF
         assert exc_info.value.strerror == os.strerror(errno.EBADF)
 
+        exc_info = raises(OSError, module.set_from_errno_special)
+        assert exc_info.value.filename == "/path/to/%s" % (char, )
+        assert exc_info.value.errno == errno.EBADF
+        assert exc_info.value.strerror == os.strerror(errno.EBADF)
+
     def test_SetFromErrnoWithFilename_NULL(self):
         import errno, os
 

From pypy.commits at gmail.com  Tue Nov  8 10:37:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 07:37:51 -0800 (PST)
Subject: [pypy-commit] pypy default: obscure,
 added a helper method called addr_add_bytes to add the offset to
 the memory location (used in vec_load/store),
 disabled some tests that are not interesting on llgraph anyway
Message-ID: <5821f14f.e39cc20a.e2b77.d82e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88221:e354d68473f3
Date: 2016-11-08 16:01 +0100
http://bitbucket.org/pypy/pypy/changeset/e354d68473f3/

Log:	obscure, added a helper method called addr_add_bytes to add the
	offset to the memory location (used in vec_load/store), disabled
	some tests that are not interesting on llgraph anyway

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -937,27 +937,26 @@
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
     def build_load(func):
-        def method(self, struct, offset, scale, disp, descr, _count):
+        def load(self, struct, offset, scale, disp, descr, _count):
             values = []
             count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
-            adr = struct + (offset * scale + disp)
+            adr = support.addr_add_bytes(struct, (offset * scale + disp))
             a = support.cast_arg(lltype.Ptr(descr.A), adr)
             array = a._obj
             for i in range(count):
                 val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
-        return method
+        return load
 
     bh_vec_load_i = build_load(bh_getarrayitem_raw)
     bh_vec_load_f = build_load(bh_getarrayitem_raw)
     del build_load
 
     def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
-        stride = descr.get_item_size_in_bytes()
-        adr = struct + (offset * scale + disp)
+        adr = support.addr_add_bytes(struct, offset * scale + disp)
         a = support.cast_arg(lltype.Ptr(descr.A), adr)
         array = a._obj
         for i,n in enumerate(newvalues):
@@ -1560,7 +1559,7 @@
             if opname.startswith('vec_'):
                 # pre vector op
                 count = self.current_op.count
-                assert count >= 1
+                assert count >= 0
                 new_args = new_args + (count,)
             result = getattr(self.cpu, 'bh_' + opname)(*new_args)
             if isinstance(result, list):
diff --git a/rpython/jit/backend/llgraph/support.py b/rpython/jit/backend/llgraph/support.py
--- a/rpython/jit/backend/llgraph/support.py
+++ b/rpython/jit/backend/llgraph/support.py
@@ -156,3 +156,11 @@
         call_args.append(n)
     assert i == len(args)
     return call_args
+
+def addr_add_bytes(addr, ofs):
+    if (isinstance(ofs, int) and
+            getattr(addr.adr.ptr._TYPE.TO, 'OF', None) == lltype.Char):
+        return addr + ofs
+    ptr = rffi.cast(rffi.CCHARP, addr.adr)
+    ptr = lltype.direct_ptradd(ptr, ofs)
+    return cast_to_int(ptr)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -75,16 +75,6 @@
     request.cls.a
     return rs
 
-
-def rdiv(v1,v2):
-    # TODO unused, interpeting this on top of llgraph does not work correctly
-    try:
-        return v1 / v2
-    except ZeroDivisionError:
-        if v1 == v2 == 0.0:
-            return rfloat.NAN
-        return rfloat.copysign(rfloat.INFINITY, v1 * v2)
-
 class VectorizeTests(object):
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -292,7 +282,7 @@
                 myjitdriver.jit_merge_point()
                 a = va[i]
                 b = vb[i]
-                ec = intmask(a) + intmask(b)
+                ec = intmask(intmask(a) + intmask(b))
                 va[i] = rffi.r_short(ec)
                 i += 1
 
@@ -544,33 +534,6 @@
         res = self.meta_interp(f, [i], vec=True)
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3),(25,2.5,0.3)])
-    def test_list_vectorize(self,i,v1,v2):
-        myjitdriver = JitDriver(greens = [],
-                                reds = 'auto')
-        class ListF(object):
-            def __init__(self, size, init):
-                self.list = [init] * size
-            def __getitem__(self, key):
-                return self.list[key]
-            def __setitem__(self, key, value):
-                self.list[key] = value
-        def f(d, v1, v2):
-            a = ListF(d, v1)
-            b = ListF(d, v2)
-            i = 0
-            while i < d:
-                myjitdriver.jit_merge_point()
-                a[i] = a[i] + b[i]
-                i += 1
-            s = 0
-            for i in range(d):
-                s += a[i]
-            return s
-        res = self.meta_interp(f, [i,v1,v2], vec=True, vec_all=True)
-        # sum helps to generate the rounding error of floating points
-        # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
-        assert res == f(i,v1,v2) == sum([v1+v2]*i)
 
     @py.test.mark.parametrize('size',[12])
     def test_body_multiple_accesses(self, size):
@@ -898,4 +861,14 @@
 
 
 class TestLLtype(LLJitMixin, VectorizeTests):
-    pass
+    # skip some tests on this backend
+    def test_unpack_f(self):
+        pass
+    def test_unpack_i64(self):
+        pass
+    def test_unpack_i(self):
+        pass
+    def test_unpack_several(self):
+        pass
+    def test_vec_int_sum(self):
+        pass

From pypy.commits at gmail.com  Tue Nov  8 10:37:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 07:37:49 -0800 (PST)
Subject: [pypy-commit] pypy default: do not call intmask on float parameters
 (as pointed out by armin)
Message-ID: <5821f14d.43921c0a.501fd.8bb8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88220:b82fe3f0417e
Date: 2016-11-08 14:21 +0100
http://bitbucket.org/pypy/pypy/changeset/b82fe3f0417e/

Log:	do not call intmask on float parameters (as pointed out by armin)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -842,16 +842,22 @@
         assert len(vx) == len(vy) == count
         return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
     """
+    vector_float_arith_code = """
+    def bh_vec_{0}_{1}(self, vx, vy, count):
+        assert len(vx) == len(vy) == count
+        return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
+    """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
     exec py.code.Source(vector_arith_code.format('int','sub','-')).compile()
     exec py.code.Source(vector_arith_code.format('int','mul','*')).compile()
     exec py.code.Source(vector_arith_code.format('int','and','&')).compile()
     exec py.code.Source(vector_arith_code.format('int','or','|')).compile()
-    exec py.code.Source(vector_arith_code.format('float','add','+')).compile()
-    exec py.code.Source(vector_arith_code.format('float','sub','-')).compile()
-    exec py.code.Source(vector_arith_code.format('float','mul','*')).compile()
-    exec py.code.Source(vector_arith_code.format('float','truediv','/')).compile()
-    exec py.code.Source(vector_arith_code.format('float','eq','==')).compile()
+
+    exec py.code.Source(vector_float_arith_code.format('float','add','+')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','sub','-')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','mul','*')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','truediv','/')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','eq','==')).compile()
 
     def bh_vec_float_neg(self, vx, count):
         return [e * -1 for e in vx]

From pypy.commits at gmail.com  Tue Nov  8 11:49:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 08:49:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Avoid calling a non-static function
 "_cpu_count()", which might conflict
Message-ID: <58220201.6249c20a.80cd6.665c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88222:27e820181ac7
Date: 2016-11-08 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/27e820181ac7/

Log:	Avoid calling a non-static function "_cpu_count()", which might
	conflict

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -2315,20 +2315,20 @@
 
 _pipe2_syscall = ENoSysCache()
 
-post_include_bits=['RPY_EXTERN int _cpu_count(void);']
+post_include_bits=['RPY_EXTERN int rpy_cpu_count(void);']
 # cpu count for linux, windows and mac (+ bsds)
 # note that the code is copied from cpython and split up here
 if sys.platform.startswith('linux'):
     cpucount_eci = ExternalCompilationInfo(includes=["unistd.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 return sysconf(_SC_NPROCESSORS_ONLN);
             }
             """], post_include_bits=post_include_bits)
 elif sys.platform == "win32":
     cpucount_eci = ExternalCompilationInfo(includes=["Windows.h"],
             separate_module_sources=["""
-        RPY_EXTERN int _cpu_count(void) {
+        RPY_EXTERN int rpy_cpu_count(void) {
             SYSTEM_INFO sysinfo;
             GetSystemInfo(&sysinfo);
             return sysinfo.dwNumberOfProcessors;
@@ -2337,7 +2337,7 @@
 else:
     cpucount_eci = ExternalCompilationInfo(includes=["sys/types.h", "sys/sysctl.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 int ncpu = 0;
             #if defined(__DragonFly__) || \
                 defined(__OpenBSD__)   || \
@@ -2355,7 +2355,7 @@
             }
             """], post_include_bits=post_include_bits)
 
-_cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real,
+_cpu_count = rffi.llexternal('rpy_cpu_count', [], rffi.INT_real,
                             compilation_info=cpucount_eci)
 
 def cpu_count():

From pypy.commits at gmail.com  Tue Nov  8 12:45:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 09:45:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Win32: fix set_inheritable tests
Message-ID: <58220f4a.c11d1c0a.93f43.b5bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88223:405237fca758
Date: 2016-11-08 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/405237fca758/

Log:	Win32: fix set_inheritable tests

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1166,6 +1166,10 @@
 @replace_os_function('pipe')
 def pipe(flags=0):
     # 'flags' might be ignored.  Check the result.
+    # The handles returned are always inheritable on Posix.
+    # The situation on Windows is not completely clear: I think
+    # it should always return non-inheritable handles, but CPython
+    # uses SECURITY_ATTRIBUTES to ensure that and we don't.
     if _WIN32:
         # 'flags' ignored
         ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
@@ -2146,8 +2150,9 @@
         handle_posix_error('mknodat', error)
 
 
-eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=[r"""
+if not _WIN32:
+    eci_inheritable = eci.merge(ExternalCompilationInfo(
+        separate_module_sources=[r"""
 #include <errno.h>
 #include <sys/ioctl.h>
 
@@ -2200,10 +2205,6 @@
 RPY_EXTERN
 int rpy_dup_noninheritable(int fd)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUPFD_CLOEXEC
     return fcntl(fd, F_DUPFD_CLOEXEC, 0);
 #else
@@ -2221,10 +2222,6 @@
 RPY_EXTERN
 int rpy_dup2_noninheritable(int fd, int fd2)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUP2FD_CLOEXEC
     return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
 
@@ -2249,33 +2246,41 @@
     return 0;
 #endif
 }
-    """ % {'HAVE_DUP3': HAVE_DUP3}],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
-                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
+        """ % {'HAVE_DUP3': HAVE_DUP3}],
+        post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                           'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n'
+                           ]))
 
-c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
-                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                compilation_info=eci_inheritable)
-c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
-                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                 compilation_info=eci_inheritable)
+    _c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    _c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                    rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                    compilation_info=eci_inheritable)
+    c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                     rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                     compilation_info=eci_inheritable)
 
-def set_inheritable(fd, inheritable):
-    result = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', result)
+    def set_inheritable(fd, inheritable):
+        result = _c_set_inheritable(fd, inheritable)
+        handle_posix_error('set_inheritable', result)
 
-def get_inheritable(fd):
-    res = c_get_inheritable(fd)
-    res = handle_posix_error('get_inheritable', res)
-    return res != 0
+    def get_inheritable(fd):
+        res = _c_get_inheritable(fd)
+        res = handle_posix_error('get_inheritable', res)
+        return res != 0
+
+else:
+    # _WIN32
+    from rpython.rlib.rwin32 import set_inheritable, get_inheritable
+    from rpython.rlib.rwin32 import c_dup_noninheritable
+    from rpython.rlib.rwin32 import c_dup2_noninheritable
+
 
 class SetNonInheritableCache(object):
     """Make one prebuilt instance of this for each path that creates
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -112,7 +112,7 @@
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
-                       STD_ERROR_HANDLE
+                       STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -473,3 +473,56 @@
     CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
     GetConsoleScreenBufferInfo = winexternal(
         "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
+
+    _GetHandleInformation = winexternal(
+        'GetHandleInformation', [HANDLE, LPDWORD], BOOL)
+    _SetHandleInformation = winexternal(
+        'SetHandleInformation', [HANDLE, DWORD, DWORD], BOOL)
+
+    def set_inheritable(fd, inheritable):
+        handle = get_osfhandle(fd)
+        if inheritable:
+            flags = HANDLE_FLAG_INHERIT
+        else:
+            flags = 0
+        if not _SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags):
+            raise lastSavedWindowsError("SetHandleInformation")
+
+    def get_inheritable(fd):
+        handle = get_osfhandle(fd)
+        pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
+        try:
+            if not _GetHandleInformation(handle, pflags):
+                raise lastSavedWindowsError("GetHandleInformation")
+            flags = pflags[0]
+        finally:
+            lltype.free(pflags, flavor='raw')
+        return (flags & HANDLE_FLAG_INHERIT) != 0
+
+    _GetFileType = winexternal('GetFileType', [HANDLE], DWORD)
+
+    def c_dup_noninheritable(fd1):
+        from rpython.rlib.rposix import c_dup
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        fd2 = c_dup(fd1)     # the inheritable version
+        if fd2 >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return fd2
+
+    def c_dup2_noninheritable(fd1, fd2):
+        from rpython.rlib.rposix import c_dup2
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        res = c_dup2(fd1, fd2)     # the inheritable version
+        if res >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return res
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -604,6 +604,9 @@
 def test_SetNonInheritableCache():
     cache = rposix.SetNonInheritableCache()
     fd1, fd2 = os.pipe()
+    if sys.platform == 'win32':
+        rposix.set_inheritable(fd1, True)
+        rposix.set_inheritable(fd2, True)
     assert rposix.get_inheritable(fd1) == True
     assert rposix.get_inheritable(fd1) == True
     assert cache.cached_inheritable == -1
@@ -616,6 +619,24 @@
     os.close(fd1)
     os.close(fd2)
 
+def test_dup_dup2_non_inheritable():
+    for preset in [False, True]:
+        fd1, fd2 = os.pipe()
+        rposix.set_inheritable(fd1, preset)
+        rposix.set_inheritable(fd2, preset)
+        fd3 = rposix.dup(fd1, True)
+        assert rposix.get_inheritable(fd3) == True
+        fd4 = rposix.dup(fd1, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd4, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd3, True)
+        assert rposix.get_inheritable(fd3) == True
+        os.close(fd1)
+        os.close(fd2)
+        os.close(fd3)
+        os.close(fd4)
+
 def test_sync():
     if sys.platform != 'win32':
         rposix.sync()

From pypy.commits at gmail.com  Tue Nov  8 13:11:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 10:11:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Win32: pass test_rposix.test_putenv
Message-ID: <58221535.ca57c20a.b99fd.eafe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88224:aa00ccf8fdaf
Date: 2016-11-08 19:09 +0100
http://bitbucket.org/pypy/pypy/changeset/aa00ccf8fdaf/

Log:	Win32: pass test_rposix.test_putenv

diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -201,6 +201,8 @@
     # default implementation for platforms without a real unsetenv()
     r_putenv(name, '')
 
+REAL_UNSETENV = False
+
 if hasattr(__import__(os.name), 'unsetenv'):
     os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], rffi.INT,
                                   save_err=rffi.RFFI_SAVE_ERRNO)
@@ -222,3 +224,4 @@
     register_external(r_unsetenv, [str0], annmodel.s_None,
                       export_name='ll_os.ll_os_unsetenv',
                       llimpl=unsetenv_llimpl)
+    REAL_UNSETENV = True
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -470,11 +470,18 @@
         assert rposix.is_valid_fd(fd) == 0
 
     def test_putenv(self):
+        from rpython.rlib import rposix_environ
+
         def f():
             rposix.putenv(self.path, self.path)
             rposix.unsetenv(self.path)
 
-        interpret(f, []) # does not crash
+        interpret(f, [],     # does not crash
+                  malloc_check=rposix_environ.REAL_UNSETENV)
+        # If we have a real unsetenv(), check that it frees the string
+        # kept alive by putenv().  Otherwise, we can't check that,
+        # because unsetenv() will keep another string alive itself.
+    test_putenv.dont_track_allocations = True
 
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):

From pypy.commits at gmail.com  Tue Nov  8 13:17:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 10:17:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Win32: remove rposix.fdlistdir()
Message-ID: <582216ae.c1341c0a.c3e5e.7671@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88225:336f8717b768
Date: 2016-11-08 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/336f8717b768/

Log:	Win32: remove rposix.fdlistdir()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -679,21 +679,22 @@
         raise OSError(error, "readdir failed")
     return result
 
-def fdlistdir(dirfd):
-    """
-    Like listdir(), except that the directory is specified as an open
-    file descriptor.
+if not _WIN32:
+    def fdlistdir(dirfd):
+        """
+        Like listdir(), except that the directory is specified as an open
+        file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.  To emulate the
-    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
-    file descriptor.
-    """
-    dirp = c_fdopendir(dirfd)
-    if not dirp:
-        error = get_saved_errno()
-        c_close(dirfd)
-        raise OSError(error, "opendir failed")
-    return _listdir(dirp, rewind=True)
+        Note: fdlistdir() closes the file descriptor.  To emulate the
+        Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+        file descriptor.
+        """
+        dirp = c_fdopendir(dirfd)
+        if not dirp:
+            error = get_saved_errno()
+            c_close(dirfd)
+            raise OSError(error, "opendir failed")
+        return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)

From pypy.commits at gmail.com  Tue Nov  8 13:38:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 10:38:00 -0800 (PST)
Subject: [pypy-commit] pypy default: We want r_longlong(42) + 0.5 to be
 42.5. Previous version returned 42.0...
Message-ID: <58221b88.88571c0a.19783.e091@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88226:a23c22636d73
Date: 2016-11-08 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/a23c22636d73/

Log:	We want r_longlong(42) + 0.5 to be 42.5. Previous version returned
	42.0...

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -323,50 +323,48 @@
 
     def __add__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x + y)
     __radd__ = __add__
 
     def __sub__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x - y)
 
     def __rsub__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # may be a float
         return self._widen(other, x - y)
 
     def __mul__(self, other):
         x = long(self)
-        if not isinstance(other, (int, long)):
-            return x * other
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x * y)
     __rmul__ = __mul__
 
     def __div__(self, other):
         x = long(self)
-        y = long(other)
-        return self._widen(other, x // y)
+        y = other      # may be a float
+        return self._widen(other, x / y)
 
     __floordiv__ = __div__
 
     def __rdiv__(self, other):
         y = long(self)
-        x = long(other)
-        return self._widen(other, x // y)
+        x = other      # may be a float
+        return self._widen(other, x / y)
 
     __rfloordiv__ = __rdiv__
 
     def __mod__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __rmod__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __divmod__(self, other):
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -602,3 +602,12 @@
 
         assert r_uint64(self._64_umax) + r_uint64(1) == r_uint64(0)
         assert r_uint64(0) - r_uint64(1) == r_uint64(self._64_umax)
+
+    def test_operation_with_float(self):
+        def f(x):
+            assert r_longlong(x) + 0.5 == 43.5
+            assert r_longlong(x) - 0.5 == 42.5
+            assert r_longlong(x) * 0.5 == 21.5
+            assert r_longlong(x) / 0.8 == 53.75
+        f(43)
+        interpret(f, [43])

From pypy.commits at gmail.com  Tue Nov  8 13:42:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:42:51 -0800 (PST)
Subject: [pypy-commit] pypy default: solved the big endian issue in the ppc
 backend
Message-ID: <58221cab.12111c0a.2882f.829f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88227:558aca7fa5b5
Date: 2016-11-08 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/558aca7fa5b5/

Log:	solved the big endian issue in the ppc backend

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -230,26 +230,19 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            for i in range(2): # at most 2 operations
-                if IS_BIG_ENDIAN:
-                    srcoff = (i * osize)
-                    resoff = (i * nsize)
-                else:
-                    if osize == 4:
-                        srcoff = 8 + (i * osize)
-                        resoff = (i * nsize)
-                    else:
-                        srcoff = (i * osize)
-                        resoff = 8 + (i * nsize)
-                if osize == 8:
-                    self.mc.load(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                else:
-                    self.mc.lwa(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                if nsize == 8:
-                    self.mc.store(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-                else:
-                    self.mc.stw(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-
+            for j in range(2): # at most 2 operations
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // osize) - 1 - i
+                off += osize * i
+                self._load_from_sp(r.SCRATCH.value, osize, off)
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // nsize) - 1 - i
+                off += nsize * i
+                self._store_to_sp(r.SCRATCH.value, nsize, off+16)
             self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
@@ -531,6 +524,37 @@
                 self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
         self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
+    def _load_from_sp(self, res, size, off):
+        if size == 8:
+            self.mc.load(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.lwa(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.lha(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.lbz(res, r.SP.value, off)
+            self.mc.extsb(res, res)
+            return True
+        return False
+
+    def _store_to_sp(self, res, size, off):
+        if size == 8:
+            self.mc.store(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.stw(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.sth(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.stz(res, r.SP.value, off)
+            return True
+        return False
+
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc, idxloc, countloc, sizeloc = arglocs
@@ -539,6 +563,7 @@
         src = srcloc.value
         size = sizeloc.value
         count = countloc.value
+        newsize = op.bytesize
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
@@ -548,18 +573,7 @@
             if not IS_BIG_ENDIAN:
                 idx = (16 // size) - 1 - idx
             off += size * idx
-            if size == 8:
-                self.mc.load(res, r.SP.value, off)
-                return
-            elif size == 4:
-                self.mc.lwa(res, r.SP.value, off)
-                return
-            elif size == 2:
-                self.mc.lha(res, r.SP.value, off)
-                return
-            elif size == 1:
-                self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
+            if self._load_from_sp(res, size, off):
                 return
         else:
             # count is not 1, but only 2 is supported for i32
@@ -571,16 +585,21 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            if count * size == 8:
+            for j in range(count):
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j+idx
                 if not IS_BIG_ENDIAN:
-                    endian_off = 8
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._load_from_sp(r.SCRATCH.value, size, off)
                 off = PARAM_SAVE_AREA_OFFSET
-                off = off + endian_off - (idx * size)
-                assert idx * size + 8 <= 16
-                self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-                return
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._store_to_sp(r.SCRATCH.value, newsize, off+16)
+            self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+            return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1397,5 +1397,6 @@
             assert op not in dups
             dups.add(op)
 
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1190,6 +1190,8 @@
     # Uh, that should be moved to vector_ext really!
     _cast_ops['CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
     _cast_ops['VEC_CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+    _cast_ops['CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
+    _cast_ops['VEC_CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -843,17 +843,18 @@
                 j += 4
                 i += 8
 
-        va = alloc_raw_storage(4*30, zero=True)
-        vb = alloc_raw_storage(8*30, zero=True)
-        for i,v in enumerate([1]*30):
+        count = 32
+        va = alloc_raw_storage(4*count, zero=True)
+        vb = alloc_raw_storage(8*count, zero=True)
+        for i,v in enumerate([1,2,3,4]*(count/4)):
             raw_storage_setitem(va, i*4, rffi.cast(rffi.INT,v))
-        for i,v in enumerate([-9.0]*30):
+        for i,v in enumerate([-1.0,-2.0,-3.0,-4.0]*(count/4)):
             raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
-        vc = alloc_raw_storage(8*30, zero=True)
-        self.meta_interp(f, [8*30, va, vb, vc], vec=True)
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
 
-        for i in range(30):
-            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == -8.0
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
 
         free_raw_storage(va)
         free_raw_storage(vb)

From pypy.commits at gmail.com  Tue Nov  8 13:42:53 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:42:53 -0800 (PST)
Subject: [pypy-commit] pypy default: another big endian issue in the ppc
 vector backend
Message-ID: <58221cad.12111c0a.6c408.e44e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88228:5ec36e7b4fc0
Date: 2016-11-08 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/5ec36e7b4fc0/

Log:	another big endian issue in the ppc vector backend

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -38,13 +38,10 @@
     # if v2 == 0 unpacks index 0 of param 2
     # if v2 == 1 unpacks index 1 of param 2
     mask = 0
-    if IS_BIG_ENDIAN:
-        not_implemented("no big endian support (yet)")
-    else:
-        if v1 == 0: mask |= 0b01
-        if v1 == 1: mask |= 0b00
-        if v2 == 0: mask |= 0b10
-        if v2 == 1: mask |= 0b00
+    if v1 == 0: mask |= 0b01
+    if v1 == 1: mask |= 0b00
+    if v2 == 0: mask |= 0b10
+    if v2 == 1: mask |= 0b00
     return mask
 
 
@@ -306,10 +303,7 @@
         acc = accumloc.value
         if arg.type == FLOAT:
             # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            self.mc.xxpermdi(tgt, acc, acc, 0b10)
             if op == '+':
                 self.mc.xsadddp(tgt, tgt, acc)
             elif op == '*':
@@ -464,23 +458,26 @@
         res, l0, off = arglocs
         size = op.bytesize
 
+        idx = 0
+        size = op.bytesize
+        if not IS_BIG_ENDIAN:
+            idx = (16 // size) - 1 - idx
+        idx *= size
+        self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
+        if size == 8:
+            idx = 1
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            idx *= size
+            self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
         self.mc.load_imm(r.SCRATCH2, off.value)
         self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
         if size == 1:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltb(res.value, res.value, 0b0000)
-            else:
-                self.mc.vspltb(res.value, res.value, 0b1111)
+            self.mc.vspltb(res.value, res.value, 0b0000)
         elif size == 2:
-            if IS_BIG_ENDIAN:
-                self.mc.vsplth(res.value, res.value, 0b000)
-            else:
-                self.mc.vsplth(res.value, res.value, 0b111)
+            self.mc.vsplth(res.value, res.value, 0b000)
         elif size == 4:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltw(res.value, res.value, 0b00)
-            else:
-                self.mc.vspltw(res.value, res.value, 0b11)
+            self.mc.vspltw(res.value, res.value, 0b00)
         elif size == 8:
             pass
         else:
@@ -489,7 +486,6 @@
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
-        srcidx = srcidxloc.value
         residx = residxloc.value
         count = countloc.value
         res = resultloc.value
@@ -551,7 +547,7 @@
             self.mc.sth(res, r.SP.value, off)
             return True
         elif size == 1:
-            self.mc.stz(res, r.SP.value, off)
+            self.mc.stb(res, r.SP.value, off)
             return True
         return False
 
@@ -610,10 +606,8 @@
         vec = vloc.value
         res = resloc.value
         src = srcloc.value
-        count = countloc.value
         residx = residxloc.value
         srcidx = srcidxloc.value
-        size = op.bytesize
         # srcloc is always a floating point register f, this means it is
         # vsr[0] == valueof(f)
         if srcidx == 0:
@@ -883,10 +877,6 @@
             mc.load_imm(l0, arg.value)
         else:
             l0 = self.ensure_reg(arg)
-        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-        size = op.bytesize
-        if size == 8:
-            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
         res = self.force_allocate_vector_reg(op)
         return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
 

From pypy.commits at gmail.com  Tue Nov  8 13:53:22 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:53:22 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: obscure,
 added a helper method called addr_add_bytes to add the offset to
 the memory location (used in vec_load/store),
 disabled some tests that are not interesting on llgraph anyway
Message-ID: <58221f22.69efc20a.b562c.4f3d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88231:cb951b3dd6f7
Date: 2016-11-08 16:01 +0100
http://bitbucket.org/pypy/pypy/changeset/cb951b3dd6f7/

Log:	obscure, added a helper method called addr_add_bytes to add the
	offset to the memory location (used in vec_load/store), disabled
	some tests that are not interesting on llgraph anyway

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -937,27 +937,26 @@
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
     def build_load(func):
-        def method(self, struct, offset, scale, disp, descr, _count):
+        def load(self, struct, offset, scale, disp, descr, _count):
             values = []
             count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
-            adr = struct + (offset * scale + disp)
+            adr = support.addr_add_bytes(struct, (offset * scale + disp))
             a = support.cast_arg(lltype.Ptr(descr.A), adr)
             array = a._obj
             for i in range(count):
                 val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
-        return method
+        return load
 
     bh_vec_load_i = build_load(bh_getarrayitem_raw)
     bh_vec_load_f = build_load(bh_getarrayitem_raw)
     del build_load
 
     def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
-        stride = descr.get_item_size_in_bytes()
-        adr = struct + (offset * scale + disp)
+        adr = support.addr_add_bytes(struct, offset * scale + disp)
         a = support.cast_arg(lltype.Ptr(descr.A), adr)
         array = a._obj
         for i,n in enumerate(newvalues):
@@ -1560,7 +1559,7 @@
             if opname.startswith('vec_'):
                 # pre vector op
                 count = self.current_op.count
-                assert count >= 1
+                assert count >= 0
                 new_args = new_args + (count,)
             result = getattr(self.cpu, 'bh_' + opname)(*new_args)
             if isinstance(result, list):
diff --git a/rpython/jit/backend/llgraph/support.py b/rpython/jit/backend/llgraph/support.py
--- a/rpython/jit/backend/llgraph/support.py
+++ b/rpython/jit/backend/llgraph/support.py
@@ -156,3 +156,11 @@
         call_args.append(n)
     assert i == len(args)
     return call_args
+
+def addr_add_bytes(addr, ofs):
+    if (isinstance(ofs, int) and
+            getattr(addr.adr.ptr._TYPE.TO, 'OF', None) == lltype.Char):
+        return addr + ofs
+    ptr = rffi.cast(rffi.CCHARP, addr.adr)
+    ptr = lltype.direct_ptradd(ptr, ofs)
+    return cast_to_int(ptr)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -75,16 +75,6 @@
     request.cls.a
     return rs
 
-
-def rdiv(v1,v2):
-    # TODO unused, interpeting this on top of llgraph does not work correctly
-    try:
-        return v1 / v2
-    except ZeroDivisionError:
-        if v1 == v2 == 0.0:
-            return rfloat.NAN
-        return rfloat.copysign(rfloat.INFINITY, v1 * v2)
-
 class VectorizeTests(object):
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -294,7 +284,7 @@
                 myjitdriver.jit_merge_point()
                 a = va[i]
                 b = vb[i]
-                ec = intmask(a) + intmask(b)
+                ec = intmask(intmask(a) + intmask(b))
                 va[i] = rffi.r_short(ec)
                 i += 1
 
@@ -546,33 +536,6 @@
         res = self.meta_interp(f, [i], vec=True)
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3),(25,2.5,0.3)])
-    def test_list_vectorize(self,i,v1,v2):
-        myjitdriver = JitDriver(greens = [],
-                                reds = 'auto')
-        class ListF(object):
-            def __init__(self, size, init):
-                self.list = [init] * size
-            def __getitem__(self, key):
-                return self.list[key]
-            def __setitem__(self, key, value):
-                self.list[key] = value
-        def f(d, v1, v2):
-            a = ListF(d, v1)
-            b = ListF(d, v2)
-            i = 0
-            while i < d:
-                myjitdriver.jit_merge_point()
-                a[i] = a[i] + b[i]
-                i += 1
-            s = 0
-            for i in range(d):
-                s += a[i]
-            return s
-        res = self.meta_interp(f, [i,v1,v2], vec=True, vec_all=True)
-        # sum helps to generate the rounding error of floating points
-        # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
-        assert res == f(i,v1,v2) == sum([v1+v2]*i)
 
     @py.test.mark.parametrize('size',[12])
     def test_body_multiple_accesses(self, size):
@@ -900,4 +863,14 @@
 
 
 class TestLLtype(LLJitMixin, VectorizeTests):
-    pass
+    # skip some tests on this backend
+    def test_unpack_f(self):
+        pass
+    def test_unpack_i64(self):
+        pass
+    def test_unpack_i(self):
+        pass
+    def test_unpack_several(self):
+        pass
+    def test_vec_int_sum(self):
+        pass

From pypy.commits at gmail.com  Tue Nov  8 13:53:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:53:18 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: add intmask to arith in
 llgraph runner (was looking at the x86 backend all the time)
Message-ID: <58221f1e.a115c20a.26a18.3487@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88229:c9be457baf05
Date: 2016-11-08 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/c9be457baf05/

Log:	add intmask to arith in llgraph runner (was looking at the x86
	backend all the time)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -18,7 +18,7 @@
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
-from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
 from rpython.rlib.objectmodel import Symbolic
 
 class LLAsmInfo(object):
@@ -840,7 +840,7 @@
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
         assert len(vx) == len(vy) == count
-        return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
+        return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
     """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
     exec py.code.Source(vector_arith_code.format('int','sub','-')).compile()
@@ -891,9 +891,6 @@
         return [longlong.getfloatstorage(float(longlong.int2singlefloat(v)))
                 for v in vx]
 
-        a = float(a)
-        return longlong.getfloatstorage(a)
-
     def bh_vec_cast_float_to_int(self, vx, count):
         return [int(x) for x in vx]
 

From pypy.commits at gmail.com  Tue Nov  8 13:53:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:53:24 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: solved the big endian issue
 in the ppc backend
Message-ID: <58221f24.88571c0a.19783.e707@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88232:d8b224b914e8
Date: 2016-11-08 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/d8b224b914e8/

Log:	solved the big endian issue in the ppc backend

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -230,26 +230,19 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            for i in range(2): # at most 2 operations
-                if IS_BIG_ENDIAN:
-                    srcoff = (i * osize)
-                    resoff = (i * nsize)
-                else:
-                    if osize == 4:
-                        srcoff = 8 + (i * osize)
-                        resoff = (i * nsize)
-                    else:
-                        srcoff = (i * osize)
-                        resoff = 8 + (i * nsize)
-                if osize == 8:
-                    self.mc.load(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                else:
-                    self.mc.lwa(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                if nsize == 8:
-                    self.mc.store(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-                else:
-                    self.mc.stw(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-
+            for j in range(2): # at most 2 operations
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // osize) - 1 - i
+                off += osize * i
+                self._load_from_sp(r.SCRATCH.value, osize, off)
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // nsize) - 1 - i
+                off += nsize * i
+                self._store_to_sp(r.SCRATCH.value, nsize, off+16)
             self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
@@ -531,6 +524,37 @@
                 self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
         self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
+    def _load_from_sp(self, res, size, off):
+        if size == 8:
+            self.mc.load(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.lwa(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.lha(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.lbz(res, r.SP.value, off)
+            self.mc.extsb(res, res)
+            return True
+        return False
+
+    def _store_to_sp(self, res, size, off):
+        if size == 8:
+            self.mc.store(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.stw(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.sth(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.stz(res, r.SP.value, off)
+            return True
+        return False
+
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc, idxloc, countloc, sizeloc = arglocs
@@ -539,6 +563,7 @@
         src = srcloc.value
         size = sizeloc.value
         count = countloc.value
+        newsize = op.bytesize
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
@@ -548,18 +573,7 @@
             if not IS_BIG_ENDIAN:
                 idx = (16 // size) - 1 - idx
             off += size * idx
-            if size == 8:
-                self.mc.load(res, r.SP.value, off)
-                return
-            elif size == 4:
-                self.mc.lwa(res, r.SP.value, off)
-                return
-            elif size == 2:
-                self.mc.lha(res, r.SP.value, off)
-                return
-            elif size == 1:
-                self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
+            if self._load_from_sp(res, size, off):
                 return
         else:
             # count is not 1, but only 2 is supported for i32
@@ -571,16 +585,21 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            if count * size == 8:
+            for j in range(count):
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j+idx
                 if not IS_BIG_ENDIAN:
-                    endian_off = 8
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._load_from_sp(r.SCRATCH.value, size, off)
                 off = PARAM_SAVE_AREA_OFFSET
-                off = off + endian_off - (idx * size)
-                assert idx * size + 8 <= 16
-                self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-                return
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._store_to_sp(r.SCRATCH.value, newsize, off+16)
+            self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+            return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1397,5 +1397,6 @@
             assert op not in dups
             dups.add(op)
 
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1190,6 +1190,8 @@
     # Uh, that should be moved to vector_ext really!
     _cast_ops['CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
     _cast_ops['VEC_CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+    _cast_ops['CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
+    _cast_ops['VEC_CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -845,17 +845,18 @@
                 j += 4
                 i += 8
 
-        va = alloc_raw_storage(4*30, zero=True)
-        vb = alloc_raw_storage(8*30, zero=True)
-        for i,v in enumerate([1]*30):
+        count = 32
+        va = alloc_raw_storage(4*count, zero=True)
+        vb = alloc_raw_storage(8*count, zero=True)
+        for i,v in enumerate([1,2,3,4]*(count/4)):
             raw_storage_setitem(va, i*4, rffi.cast(rffi.INT,v))
-        for i,v in enumerate([-9.0]*30):
+        for i,v in enumerate([-1.0,-2.0,-3.0,-4.0]*(count/4)):
             raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
-        vc = alloc_raw_storage(8*30, zero=True)
-        self.meta_interp(f, [8*30, va, vb, vc], vec=True)
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
 
-        for i in range(30):
-            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == -8.0
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
 
         free_raw_storage(va)
         free_raw_storage(vb)

From pypy.commits at gmail.com  Tue Nov  8 13:53:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:53:25 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: another big endian issue in
 the ppc vector backend
Message-ID: <58221f25.233ac20a.0b5a.3ba5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88233:575021da212c
Date: 2016-11-08 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/575021da212c/

Log:	another big endian issue in the ppc vector backend

diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -38,13 +38,10 @@
     # if v2 == 0 unpacks index 0 of param 2
     # if v2 == 1 unpacks index 1 of param 2
     mask = 0
-    if IS_BIG_ENDIAN:
-        not_implemented("no big endian support (yet)")
-    else:
-        if v1 == 0: mask |= 0b01
-        if v1 == 1: mask |= 0b00
-        if v2 == 0: mask |= 0b10
-        if v2 == 1: mask |= 0b00
+    if v1 == 0: mask |= 0b01
+    if v1 == 1: mask |= 0b00
+    if v2 == 0: mask |= 0b10
+    if v2 == 1: mask |= 0b00
     return mask
 
 
@@ -306,10 +303,7 @@
         acc = accumloc.value
         if arg.type == FLOAT:
             # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            self.mc.xxpermdi(tgt, acc, acc, 0b10)
             if op == '+':
                 self.mc.xsadddp(tgt, tgt, acc)
             elif op == '*':
@@ -464,23 +458,26 @@
         res, l0, off = arglocs
         size = op.bytesize
 
+        idx = 0
+        size = op.bytesize
+        if not IS_BIG_ENDIAN:
+            idx = (16 // size) - 1 - idx
+        idx *= size
+        self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
+        if size == 8:
+            idx = 1
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            idx *= size
+            self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
         self.mc.load_imm(r.SCRATCH2, off.value)
         self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
         if size == 1:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltb(res.value, res.value, 0b0000)
-            else:
-                self.mc.vspltb(res.value, res.value, 0b1111)
+            self.mc.vspltb(res.value, res.value, 0b0000)
         elif size == 2:
-            if IS_BIG_ENDIAN:
-                self.mc.vsplth(res.value, res.value, 0b000)
-            else:
-                self.mc.vsplth(res.value, res.value, 0b111)
+            self.mc.vsplth(res.value, res.value, 0b000)
         elif size == 4:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltw(res.value, res.value, 0b00)
-            else:
-                self.mc.vspltw(res.value, res.value, 0b11)
+            self.mc.vspltw(res.value, res.value, 0b00)
         elif size == 8:
             pass
         else:
@@ -489,7 +486,6 @@
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
-        srcidx = srcidxloc.value
         residx = residxloc.value
         count = countloc.value
         res = resultloc.value
@@ -551,7 +547,7 @@
             self.mc.sth(res, r.SP.value, off)
             return True
         elif size == 1:
-            self.mc.stz(res, r.SP.value, off)
+            self.mc.stb(res, r.SP.value, off)
             return True
         return False
 
@@ -610,10 +606,8 @@
         vec = vloc.value
         res = resloc.value
         src = srcloc.value
-        count = countloc.value
         residx = residxloc.value
         srcidx = srcidxloc.value
-        size = op.bytesize
         # srcloc is always a floating point register f, this means it is
         # vsr[0] == valueof(f)
         if srcidx == 0:
@@ -883,10 +877,6 @@
             mc.load_imm(l0, arg.value)
         else:
             l0 = self.ensure_reg(arg)
-        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-        size = op.bytesize
-        if size == 8:
-            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
         res = self.force_allocate_vector_reg(op)
         return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
 

From pypy.commits at gmail.com  Tue Nov  8 13:53:20 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 10:53:20 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: do not call intmask on
 float parameters (as pointed out by armin)
Message-ID: <58221f20.ca57c20a.b99fd.fe17@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88230:f7ba891dd786
Date: 2016-11-08 14:21 +0100
http://bitbucket.org/pypy/pypy/changeset/f7ba891dd786/

Log:	do not call intmask on float parameters (as pointed out by armin)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -842,16 +842,22 @@
         assert len(vx) == len(vy) == count
         return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
     """
+    vector_float_arith_code = """
+    def bh_vec_{0}_{1}(self, vx, vy, count):
+        assert len(vx) == len(vy) == count
+        return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
+    """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
     exec py.code.Source(vector_arith_code.format('int','sub','-')).compile()
     exec py.code.Source(vector_arith_code.format('int','mul','*')).compile()
     exec py.code.Source(vector_arith_code.format('int','and','&')).compile()
     exec py.code.Source(vector_arith_code.format('int','or','|')).compile()
-    exec py.code.Source(vector_arith_code.format('float','add','+')).compile()
-    exec py.code.Source(vector_arith_code.format('float','sub','-')).compile()
-    exec py.code.Source(vector_arith_code.format('float','mul','*')).compile()
-    exec py.code.Source(vector_arith_code.format('float','truediv','/')).compile()
-    exec py.code.Source(vector_arith_code.format('float','eq','==')).compile()
+
+    exec py.code.Source(vector_float_arith_code.format('float','add','+')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','sub','-')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','mul','*')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','truediv','/')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','eq','==')).compile()
 
     def bh_vec_float_neg(self, vx, count):
         return [e * -1 for e in vx]

From pypy.commits at gmail.com  Tue Nov  8 14:02:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 11:02:57 -0800 (PST)
Subject: [pypy-commit] pypy default: Win32: fix making sockets
 non-inheritable
Message-ID: <58222161.41d71c0a.85aa6.90e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88234:2e2a6396f7c9
Date: 2016-11-08 20:02 +0100
http://bitbucket.org/pypy/pypy/changeset/2e2a6396f7c9/

Log:	Win32: fix making sockets non-inheritable

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -641,6 +641,7 @@
 
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
+        # XXX 'traverse' is ignored
         win32traits = make_win32_traits(traits)
         with lltype.scoped_alloc(
                 win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data:
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1062,11 +1062,32 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
-def sock_set_inheritable(fd, inheritable):
-    try:
-        rposix.set_inheritable(fd, inheritable)
-    except OSError as e:
-        raise CSocketError(e.errno)
+if _c.WIN32:
+    def sock_set_inheritable(fd, inheritable):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            rwin32.set_handle_inheritable(handle, inheritable)
+        except WindowsError:
+            raise RSocketError("SetHandleInformation failed")   # xxx
+
+    def sock_get_inheritable(fd):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            return rwin32.get_handle_inheritable(handle)
+        except WindowsError:
+            raise RSocketError("GetHandleInformation failed")   # xxx
+else:
+    def sock_set_inheritable(fd, inheritable):
+        try:
+            rposix.set_inheritable(fd, inheritable)
+        except OSError as e:
+            raise CSocketError(e.errno)
+
+    def sock_get_inheritable(fd):
+        try:
+            return rposix.get_inheritable(fd)
+        except OSError as e:
+            raise CSocketError(e.errno)
 
 class SocketError(Exception):
     applevelerrcls = 'error'
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -481,6 +481,10 @@
 
     def set_inheritable(fd, inheritable):
         handle = get_osfhandle(fd)
+        set_handle_inheritable(handle, inheritable)
+
+    def set_handle_inheritable(handle, inheritable):
+        assert lltype.typeOf(handle) is HANDLE
         if inheritable:
             flags = HANDLE_FLAG_INHERIT
         else:
@@ -490,6 +494,10 @@
 
     def get_inheritable(fd):
         handle = get_osfhandle(fd)
+        return get_handle_inheritable(handle)
+
+    def get_handle_inheritable(handle):
+        assert lltype.typeOf(handle) is HANDLE
         pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
         try:
             if not _GetHandleInformation(handle, pflags):
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -124,8 +124,8 @@
         py.test.skip('No socketpair on Windows')
     for inh in [False, True]:
         s1, s2 = socketpair(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
-        assert rposix.get_inheritable(s2.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s2.fd) == inh
         s1.close()
         s2.close()
 
@@ -391,7 +391,7 @@
 def test_inheritable():
     for inh in [False, True]:
         s1 = RSocket(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
         s1.close()
 
 def test_getaddrinfo_http():

From pypy.commits at gmail.com  Tue Nov  8 14:14:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 11:14:47 -0800 (PST)
Subject: [pypy-commit] pypy default: account for vector guards while parsing
 the textual jit log
Message-ID: <58222427.04831c0a.9ec54.f0fc@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88235:0c1ef72f23c4
Date: 2016-11-08 20:13 +0100
http://bitbucket.org/pypy/pypy/changeset/0c1ef72f23c4/

Log:	account for vector guards while parsing the textual jit log

diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -62,7 +62,7 @@
         guard_dict = {}
         for loop_no, loop in enumerate(loops):
             for op in loop.operations:
-                if op.name.startswith('guard_'):
+                if op.name.startswith('guard_') or op.name.startswith('vec_guard_'):
                     guard_dict[int(op.descr[len('<Guard0x'):-1], 16)] = (op, loop)
         for loop in loops:
             if loop.comment:

From pypy.commits at gmail.com  Tue Nov  8 14:23:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 11:23:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Win32: pass/skip a few tests
Message-ID: <58222619.ca57c20a.b99fd.0ad1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88236:3f7920856747
Date: 2016-11-08 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/3f7920856747/

Log:	Win32: pass/skip a few tests

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1157,12 +1157,19 @@
     def setup_class(cls):
         cls.w_path = space.wrap(str(path))
 
-    def test_environ(self):
-        import posix
-        assert posix.environ['PATH']
-        del posix.environ['PATH']
-        def fn(): posix.environ['PATH']
-        raises(KeyError, fn)
+    if sys.platform != 'win32':
+        def test_environ(self):
+            import posix
+            assert posix.environ['PATH']
+            del posix.environ['PATH']
+            def fn(): posix.environ['PATH']
+            raises(KeyError, fn)
+    else:
+        def test_environ(self):
+            import nt
+            assert 'ADLDJSSLDFKJSD' not in nt.environ
+            def fn(): nt.environ['ADLDJSSLDFKJSD']
+            raises(KeyError, fn)
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
@@ -1184,6 +1191,8 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
+        if sys.platform == 'win32':
+            py.test.skip("Posix-only tests")
         if cls.runappdirect:
             # Can't change encoding
             try:

From pypy.commits at gmail.com  Tue Nov  8 14:38:08 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 11:38:08 -0800 (PST)
Subject: [pypy-commit] pypy default: two ppc fixes (test_form, test_ppc)
Message-ID: <582229a0.060ec20a.9d15b.4ff3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88237:5860f448a03f
Date: 2016-11-08 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/5860f448a03f/

Log:	two ppc fixes (test_form, test_ppc)

diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -28,18 +28,18 @@
     w = a.insts[-1].assemble()
     return struct.pack('>i', w)
 
-class TestForm(Form):
+class TForm(Form):
     fieldmap = test_fieldmap
 
 class TestForms(object):
     def test_bitclash(self):
-        raises(FormException, TestForm, 'h', 'hh')
-        raises(FormException, TestForm,
+        raises(FormException, TForm, 'h', 'hh')
+        raises(FormException, TForm,
                Field('t1', 0, 0), Field('t2', 0, 0))
 
     def test_basic(self):
         class T(Assembler):
-            i = TestForm('h', 'l')()
+            i = TForm('h', 'l')()
             j = i(h=1)
             k = i(l=3)
             raises(FormException, k, l=0)
@@ -56,7 +56,7 @@
 
     def test_defdesc(self):
         class T(Assembler):
-            i = TestForm('hh', 'hl', 'lh', 'll')()
+            i = TForm('hh', 'hl', 'lh', 'll')()
             i.default(hl=0).default(hh=1)
             insts = []
         a = T()
diff --git a/rpython/jit/backend/ppc/test/test_ppc.py b/rpython/jit/backend/ppc/test/test_ppc.py
--- a/rpython/jit/backend/ppc/test/test_ppc.py
+++ b/rpython/jit/backend/ppc/test/test_ppc.py
@@ -199,9 +199,9 @@
             a.load_imm(r10, call_addr)
         elif IS_BIG_ENDIAN:
             # load the 3-words descriptor
-            a.load_from_addr(r10, call_addr)
-            a.load_from_addr(r2, call_addr+WORD)
-            a.load_from_addr(r11, call_addr+2*WORD)
+            a.load_from_addr(r10, SCRATCH2, call_addr)
+            a.load_from_addr(r2, SCRATCH2, call_addr+WORD)
+            a.load_from_addr(r11, SCRATCH2, call_addr+2*WORD)
         else:
             # no descriptor on little-endian, but the ABI says r12 must
             # contain the function pointer
@@ -304,7 +304,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from_addr(r3, addr)
+        a.load_from_addr(r3, SCRATCH2, addr)
         a.blr()
         f = a.get_assembler_function()
         assert f() == 200

From pypy.commits at gmail.com  Tue Nov  8 14:52:48 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 08 Nov 2016 11:52:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Make sure that the response gets closed,
 to prevent deadlock
Message-ID: <58222d10.e626c20a.4cc77.4eb3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88238:60632ceee47b
Date: 2016-11-08 19:49 +0000
http://bitbucket.org/pypy/pypy/changeset/60632ceee47b/

Log:	Make sure that the response gets closed, to prevent deadlock

diff --git a/lib-python/3/test/test_httpservers.py b/lib-python/3/test/test_httpservers.py
--- a/lib-python/3/test/test_httpservers.py
+++ b/lib-python/3/test/test_httpservers.py
@@ -254,7 +254,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('GET', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         self.assertTrue(
             err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n'))
@@ -265,7 +266,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('ERROR', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         lines = err.getvalue().split('\n')
         self.assertTrue(lines[0].endswith('code 404, message File not found'))

From pypy.commits at gmail.com  Tue Nov  8 14:54:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 11:54:16 -0800 (PST)
Subject: [pypy-commit] pypy default: remove unused imports in test file
Message-ID: <58222d68.045d1c0a.b23cf.a3c5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88239:5d97e67dcc3c
Date: 2016-11-08 20:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5d97e67dcc3c/

Log:	remove unused imports in test file

diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -1,6 +1,3 @@
-from rpython.jit.backend.ppc.codebuilder import b
-import random
-import sys
 from py.test import raises
 
 from rpython.jit.backend.ppc.form import Form, FormException

From pypy.commits at gmail.com  Tue Nov  8 14:54:17 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 11:54:17 -0800 (PST)
Subject: [pypy-commit] pypy default: int_floordiv/int_mod is not a
 resoperation anymore, replace those operations in two ppc regalloc tests
Message-ID: <58222d69.46bb1c0a.31433.fdd2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88240:ca6dfb1e30b8
Date: 2016-11-08 20:53 +0100
http://bitbucket.org/pypy/pypy/changeset/ca6dfb1e30b8/

Log:	int_floordiv/int_mod is not a resoperation anymore, replace those
	operations in two ppc regalloc tests

diff --git a/rpython/jit/backend/ppc/test/test_regalloc_3.py b/rpython/jit/backend/ppc/test/test_regalloc_3.py
--- a/rpython/jit/backend/ppc/test/test_regalloc_3.py
+++ b/rpython/jit/backend/ppc/test/test_regalloc_3.py
@@ -55,7 +55,7 @@
     i22 = int_or(i3, i9)
     i41 = int_and(i11, -4)
     i42 = int_or(i41, 1)
-    i23 = int_mod(i12, i42)
+    i23 = int_mul(i12, 0)
     i24 = int_is_true(i6)
     i25 = uint_rshift(i15, 6)
     i26 = int_or(-4, i25)
@@ -63,11 +63,11 @@
     i28 = int_sub(-113, i11)
     i29 = int_neg(i7)
     i30 = int_neg(i24)
-    i31 = int_floordiv(i3, 53)
+    i31 = int_mul(i3, 53)
     i32 = int_mul(i28, i27)
     i43 = int_and(i18, -4)
     i44 = int_or(i43, 1)
-    i33 = int_mod(i26, i44)
+    i33 = int_mul(i26, i44)
     i34 = int_or(i27, i19)
     i35 = uint_lt(i13, 1)
     i45 = int_and(i21, 31)
@@ -84,7 +84,7 @@
     assert cpu.get_int_value(deadframe, 0) == 0
     assert cpu.get_int_value(deadframe, 1) == 0
     assert cpu.get_int_value(deadframe, 2) == 0
-    assert cpu.get_int_value(deadframe, 3) == 0
+    assert cpu.get_int_value(deadframe, 3) == 530
     assert cpu.get_int_value(deadframe, 4) == 1
     assert cpu.get_int_value(deadframe, 5) == -7
     assert cpu.get_int_value(deadframe, 6) == 1
@@ -94,7 +94,7 @@
     assert cpu.get_int_value(deadframe, 10) == 1
     assert cpu.get_int_value(deadframe, 11) == 18
     assert cpu.get_int_value(deadframe, 12) == -1
-    assert cpu.get_int_value(deadframe, 13) == 0
+    assert cpu.get_int_value(deadframe, 13) == 1
 
 def test_bug_1():
     cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], '''
@@ -112,7 +112,7 @@
     i43 = int_and(i14, 31)
     i19 = int_lshift(7, i43)
     i20 = int_neg(i19)
-    i21 = int_mod(i3, 1)
+    i21 = int_and(i3, 0)
     i22 = uint_ge(i15, i1)
     i44 = int_and(i16, 31)
     i23 = int_lshift(i8, i44)

From pypy.commits at gmail.com  Tue Nov  8 15:22:53 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 12:22:53 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: two ppc fixes (test_form,
 test_ppc)
Message-ID: <5822341d.913fc20a.fd955.67d3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88242:4a74144afadb
Date: 2016-11-08 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/4a74144afadb/

Log:	two ppc fixes (test_form, test_ppc)

diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -28,18 +28,18 @@
     w = a.insts[-1].assemble()
     return struct.pack('>i', w)
 
-class TestForm(Form):
+class TForm(Form):
     fieldmap = test_fieldmap
 
 class TestForms(object):
     def test_bitclash(self):
-        raises(FormException, TestForm, 'h', 'hh')
-        raises(FormException, TestForm,
+        raises(FormException, TForm, 'h', 'hh')
+        raises(FormException, TForm,
                Field('t1', 0, 0), Field('t2', 0, 0))
 
     def test_basic(self):
         class T(Assembler):
-            i = TestForm('h', 'l')()
+            i = TForm('h', 'l')()
             j = i(h=1)
             k = i(l=3)
             raises(FormException, k, l=0)
@@ -56,7 +56,7 @@
 
     def test_defdesc(self):
         class T(Assembler):
-            i = TestForm('hh', 'hl', 'lh', 'll')()
+            i = TForm('hh', 'hl', 'lh', 'll')()
             i.default(hl=0).default(hh=1)
             insts = []
         a = T()
diff --git a/rpython/jit/backend/ppc/test/test_ppc.py b/rpython/jit/backend/ppc/test/test_ppc.py
--- a/rpython/jit/backend/ppc/test/test_ppc.py
+++ b/rpython/jit/backend/ppc/test/test_ppc.py
@@ -199,9 +199,9 @@
             a.load_imm(r10, call_addr)
         elif IS_BIG_ENDIAN:
             # load the 3-words descriptor
-            a.load_from_addr(r10, call_addr)
-            a.load_from_addr(r2, call_addr+WORD)
-            a.load_from_addr(r11, call_addr+2*WORD)
+            a.load_from_addr(r10, SCRATCH2, call_addr)
+            a.load_from_addr(r2, SCRATCH2, call_addr+WORD)
+            a.load_from_addr(r11, SCRATCH2, call_addr+2*WORD)
         else:
             # no descriptor on little-endian, but the ABI says r12 must
             # contain the function pointer
@@ -304,7 +304,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from_addr(r3, addr)
+        a.load_from_addr(r3, SCRATCH2, addr)
         a.blr()
         f = a.get_assembler_function()
         assert f() == 200

From pypy.commits at gmail.com  Tue Nov  8 15:22:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 12:22:54 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: remove unused imports in
 test file
Message-ID: <5822341e.cf3fc20a.58932.cb1b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88243:5b71fa21dc86
Date: 2016-11-08 20:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5b71fa21dc86/

Log:	remove unused imports in test file

diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -1,6 +1,3 @@
-from rpython.jit.backend.ppc.codebuilder import b
-import random
-import sys
 from py.test import raises
 
 from rpython.jit.backend.ppc.form import Form, FormException

From pypy.commits at gmail.com  Tue Nov  8 15:22:56 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 08 Nov 2016 12:22:56 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: int_floordiv/int_mod is not
 a resoperation anymore, replace those operations in two ppc regalloc tests
Message-ID: <58223420.cf3fc20a.58932.cb1d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88244:5d6f1ada014a
Date: 2016-11-08 20:53 +0100
http://bitbucket.org/pypy/pypy/changeset/5d6f1ada014a/

Log:	int_floordiv/int_mod is not a resoperation anymore, replace those
	operations in two ppc regalloc tests

diff --git a/rpython/jit/backend/ppc/test/test_regalloc_3.py b/rpython/jit/backend/ppc/test/test_regalloc_3.py
--- a/rpython/jit/backend/ppc/test/test_regalloc_3.py
+++ b/rpython/jit/backend/ppc/test/test_regalloc_3.py
@@ -55,7 +55,7 @@
     i22 = int_or(i3, i9)
     i41 = int_and(i11, -4)
     i42 = int_or(i41, 1)
-    i23 = int_mod(i12, i42)
+    i23 = int_mul(i12, 0)
     i24 = int_is_true(i6)
     i25 = uint_rshift(i15, 6)
     i26 = int_or(-4, i25)
@@ -63,11 +63,11 @@
     i28 = int_sub(-113, i11)
     i29 = int_neg(i7)
     i30 = int_neg(i24)
-    i31 = int_floordiv(i3, 53)
+    i31 = int_mul(i3, 53)
     i32 = int_mul(i28, i27)
     i43 = int_and(i18, -4)
     i44 = int_or(i43, 1)
-    i33 = int_mod(i26, i44)
+    i33 = int_mul(i26, i44)
     i34 = int_or(i27, i19)
     i35 = uint_lt(i13, 1)
     i45 = int_and(i21, 31)
@@ -84,7 +84,7 @@
     assert cpu.get_int_value(deadframe, 0) == 0
     assert cpu.get_int_value(deadframe, 1) == 0
     assert cpu.get_int_value(deadframe, 2) == 0
-    assert cpu.get_int_value(deadframe, 3) == 0
+    assert cpu.get_int_value(deadframe, 3) == 530
     assert cpu.get_int_value(deadframe, 4) == 1
     assert cpu.get_int_value(deadframe, 5) == -7
     assert cpu.get_int_value(deadframe, 6) == 1
@@ -94,7 +94,7 @@
     assert cpu.get_int_value(deadframe, 10) == 1
     assert cpu.get_int_value(deadframe, 11) == 18
     assert cpu.get_int_value(deadframe, 12) == -1
-    assert cpu.get_int_value(deadframe, 13) == 0
+    assert cpu.get_int_value(deadframe, 13) == 1
 
 def test_bug_1():
     cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], '''
@@ -112,7 +112,7 @@
     i43 = int_and(i14, 31)
     i19 = int_lshift(7, i43)
     i20 = int_neg(i19)
-    i21 = int_mod(i3, 1)
+    i21 = int_and(i3, 0)
     i22 = uint_ge(i15, i1)
     i44 = int_and(i16, 31)
     i23 = int_lshift(i8, i44)

From pypy.commits at gmail.com  Tue Nov  8 15:22:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 08 Nov 2016 12:22:51 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: fix test
Message-ID: <5822341b.c1d81c0a.b3bbe.b306@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88241:a247645e7d20
Date: 2016-11-08 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/a247645e7d20/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -418,7 +418,7 @@
         guard_no_exception(descr=...)
         i112 = int_signext(i160, 2)
         setfield_gc(p167, ConstPtr(ptr85), descr=<FieldP pypy.module._cffi_backend.cdataobj.W_CData.inst_ctype .+>)
-        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_length .+>)
+        setfield_gc(p167, -1, descr=<FieldS pypy.module._cffi_backend.cdataobj.W_CDataNewOwning.inst_allocated_length .+>)
         i114 = int_ne(i160, i112)
         guard_false(i114, descr=...)
         --TICK--

From pypy.commits at gmail.com  Wed Nov  9 01:22:36 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 08 Nov 2016 22:22:36 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: add more userslot functions,
 cleanup slotdefs for repeated definitions of __getattr__
Message-ID: <5822c0ac.581d1c0a.bc46b.acfb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88245:1b0451031b2e
Date: 2016-11-02 00:32 +0200
http://bitbucket.org/pypy/pypy/changeset/1b0451031b2e/

Log:	add more userslot functions, cleanup slotdefs for repeated
	definitions of __getattr__

diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -64,6 +64,7 @@
 #   error "Python needs a typedef for Py_uintptr_t in pyport.h."
 #endif /* HAVE_UINTPTR_T */
 
+#include <time.h>
 
 /*******************************
  * stat() and fstat() fiddling *
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -748,7 +748,7 @@
 static slotdef slotdefs[] = {
         SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc,
                "x.__len__() <==> len(x)"),
-        SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc,
+        SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc,
           "x.__add__(y) <==> x+y"),
         SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc,
           "x.__mul__(n) <==> x*n"),
@@ -896,9 +896,7 @@
                "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS),
         TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook,
                wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"),
-        TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""),
-        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""),
-        TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""),
+        TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""),
         TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr,
                "x.__setattr__('name', value) <==> x.name = value"),
         TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""),
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -14,23 +14,86 @@
 from pypy.interpreter.argument import Arguments
 from pypy.module.cpyext.api import cpython_api, PyObject, Py_ssize_t
 from pypy.module.cpyext.api import PyTypeObjectPtr
-
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 @cpython_api([PyObject], Py_ssize_t, error=-1, header=None)
 def slot_sq_length(space, w_obj):
     return space.int_w(space.len(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, header=None, error=-1)
+def slot_tp_hash(space, w_obj):
+    return space.int_w(space.hash(w_obj))
+
 @cpython_api([PyObject, Py_ssize_t], PyObject, header=None)
 def slot_sq_item(space, w_obj, index):
     return space.getitem(w_obj, space.wrap(index))
 
+ at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
+def slot_tp_new(space, w_type, w_args, w_kwds):
+    w_impl = space.getattr(w_type, space.wrap('__new__'))
+    import pdb;pdb.set_trace()
+    args = Arguments(space, [w_type],
+                     w_stararg=w_args, w_starstararg=w_kwds)
+    return space.call_args(w_impl, args)
+
+# unary functions
+
+ at cpython_api([PyObject], PyObject, header=None)
+def slot_tp_str(space, w_obj):
+    return space.str(w_obj)
+
+ at cpython_api([PyObject], PyObject, header=None)
+def slot_tp_repr(space, w_obj):
+    return space.repr(w_obj)
+
+#binary functions
+
 @cpython_api([PyObject, PyObject], PyObject, header=None)
 def slot_nb_add(space, w_obj1, w_obj2):
     return space.add(w_obj1, w_obj2)
 
- at cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
-def slot_tp_new(space, w_type, w_args, w_kwds):
-    w_impl = space.getattr(w_type, space.wrap('__new__'))
-    args = Arguments(space, [w_type],
-                     w_stararg=w_args, w_starstararg=w_kwds)
-    return space.call_args(w_impl, args)
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_subtract(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_multiply(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_divide(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_inplace_add(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_inplace_subtract(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_inplace_multiply(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_nb_inplace_divide(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_sq_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_sq_inplace_concat(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_mp_subscript(space, w_obj1, w_obj2):
+    return space.add(w_obj1, w_obj2)
+
+ at cpython_api([PyObject, PyObject], PyObject, header=None)
+def slot_tp_getattr(space, w_obj1, w_obj2):
+    return space.getattr(w_obj1, w_obj2)
+
+

From pypy.commits at gmail.com  Wed Nov  9 01:22:38 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 08 Nov 2016 22:22:38 -0800 (PST)
Subject: [pypy-commit] pypy default: update repackage script
Message-ID: <5822c0ae.231cc20a.37924.fbcc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88246:e0419340cee7
Date: 2016-11-09 08:21 +0200
http://bitbucket.org/pypy/pypy/changeset/e0419340cee7/

Log:	update repackage script

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=4
-rev=1
+min=6
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Wed Nov  9 03:41:34 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 00:41:34 -0800 (PST)
Subject: [pypy-commit] pypy default: add one test and add signextenstion
 needed for one case (s390x)
Message-ID: <5822e13e.43e61c0a.3cdac.e532@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88247:ce81e2bb2517
Date: 2016-11-09 09:40 +0100
http://bitbucket.org/pypy/pypy/changeset/ce81e2bb2517/

Log:	add one test and add signextenstion needed for one case (s390x)

diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -76,7 +76,6 @@
     'CLGRJ':   ('rie_b',  ['\xEC','\x65']),
     # compare and swap
     'CSG':     ('rsy_a', ['\xEB','\x30']),
-
 }
 
 logic_mnemonic_codes = {
@@ -340,7 +339,7 @@
     'VLVG':  ('vrs_b', ['\xE7','\x22']),
     'VLGV':  ('vrs_c', ['\xE7','\x21']),
 
-    # '': ('', ['','']),
+    'VSEG':  ('vrr_a', ['\xE7','\x5F'], 'v,v,m'),
 }
 
 all_mnemonic_codes.update(arith_mnemonic_codes)
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -139,6 +139,8 @@
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(0), l.itemsize_to_mask(nsize))
             self.mc.VLGV(r.SCRATCH, loc0, l.addr(1), l.itemsize_to_mask(osize))
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(1), l.itemsize_to_mask(nsize))
+            if nsize == 8:
+                self.mc.VSEG(resloc, resloc, l.itemsize_to_mask(osize))
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -14,6 +14,7 @@
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
 
 
 def forwarded_vecinfo(op):
@@ -315,11 +316,8 @@
 
 def failnbail_transformation(msg):
     msg = '%s\n' % msg
-    if we_are_translated():
-        llop.debug_print(lltype.Void, msg)
-    else:
-        import pdb; pdb.set_trace()
-    raise NotImplementedError(msg)
+    debug_print(msg)
+    raise NotAVectorizeableLoop
 
 def turn_into_vector(state, pack):
     """ Turn a pack into a vector instruction """
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -860,6 +860,37 @@
         free_raw_storage(vb)
         free_raw_storage(vc)
 
+    def test_float_int32_casts(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            j = 0
+            while i < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(rffi.DOUBLE,va,j)
+                b = raw_storage_getitem(rffi.INT,vb,i)
+                c = a+rffi.cast(rffi.DOUBLE,b)
+                raw_storage_setitem(vc, j, c)
+                i += 4
+                j += 8
+
+        count = 32
+        va = alloc_raw_storage(8*count, zero=True)
+        vb = alloc_raw_storage(4*count, zero=True)
+        for i,v in enumerate([1.0,2.0,3.0,4.0]*(count/4)):
+            raw_storage_setitem(va, i*8, rffi.cast(rffi.DOUBLE,v))
+        for i,v in enumerate([-1,-2,-3,-4]*(count/4)):
+            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
+
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
+
+        free_raw_storage(va)
+        free_raw_storage(vb)
+        free_raw_storage(vc)
+
 
 class TestLLtype(LLJitMixin, VectorizeTests):
     # skip some tests on this backend

From pypy.commits at gmail.com  Wed Nov  9 03:50:37 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 Nov 2016 00:50:37 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: update repackage script
Message-ID: <5822e35d.88051c0a.314c.de90@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88248:6b7427d2e889
Date: 2016-11-09 08:21 +0200
http://bitbucket.org/pypy/pypy/changeset/6b7427d2e889/

Log:	update repackage script

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=4
-rev=1
+min=6
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 

From pypy.commits at gmail.com  Wed Nov  9 03:50:39 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 00:50:39 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: add one test and add
 signextenstion needed for one case (s390x)
Message-ID: <5822e35f.13d41c0a.1182.8394@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88249:9ffb23134573
Date: 2016-11-09 09:40 +0100
http://bitbucket.org/pypy/pypy/changeset/9ffb23134573/

Log:	add one test and add signextenstion needed for one case (s390x)

diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -76,7 +76,6 @@
     'CLGRJ':   ('rie_b',  ['\xEC','\x65']),
     # compare and swap
     'CSG':     ('rsy_a', ['\xEB','\x30']),
-
 }
 
 logic_mnemonic_codes = {
@@ -340,7 +339,7 @@
     'VLVG':  ('vrs_b', ['\xE7','\x22']),
     'VLGV':  ('vrs_c', ['\xE7','\x21']),
 
-    # '': ('', ['','']),
+    'VSEG':  ('vrr_a', ['\xE7','\x5F'], 'v,v,m'),
 }
 
 all_mnemonic_codes.update(arith_mnemonic_codes)
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -139,6 +139,8 @@
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(0), l.itemsize_to_mask(nsize))
             self.mc.VLGV(r.SCRATCH, loc0, l.addr(1), l.itemsize_to_mask(osize))
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(1), l.itemsize_to_mask(nsize))
+            if nsize == 8:
+                self.mc.VSEG(resloc, resloc, l.itemsize_to_mask(osize))
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -14,6 +14,7 @@
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
 
 
 def forwarded_vecinfo(op):
@@ -315,11 +316,8 @@
 
 def failnbail_transformation(msg):
     msg = '%s\n' % msg
-    if we_are_translated():
-        llop.debug_print(lltype.Void, msg)
-    else:
-        import pdb; pdb.set_trace()
-    raise NotImplementedError(msg)
+    debug_print(msg)
+    raise NotAVectorizeableLoop
 
 def turn_into_vector(state, pack):
     """ Turn a pack into a vector instruction """
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -862,6 +862,37 @@
         free_raw_storage(vb)
         free_raw_storage(vc)
 
+    def test_float_int32_casts(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            j = 0
+            while i < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(rffi.DOUBLE,va,j)
+                b = raw_storage_getitem(rffi.INT,vb,i)
+                c = a+rffi.cast(rffi.DOUBLE,b)
+                raw_storage_setitem(vc, j, c)
+                i += 4
+                j += 8
+
+        count = 32
+        va = alloc_raw_storage(8*count, zero=True)
+        vb = alloc_raw_storage(4*count, zero=True)
+        for i,v in enumerate([1.0,2.0,3.0,4.0]*(count/4)):
+            raw_storage_setitem(va, i*8, rffi.cast(rffi.DOUBLE,v))
+        for i,v in enumerate([-1,-2,-3,-4]*(count/4)):
+            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
+
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
+
+        free_raw_storage(va)
+        free_raw_storage(vb)
+        free_raw_storage(vc)
+
 
 class TestLLtype(LLJitMixin, VectorizeTests):
     # skip some tests on this backend

From pypy.commits at gmail.com  Wed Nov  9 03:50:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 00:50:41 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Win32: pass/skip a few tests
Message-ID: <5822e361.4b8f1c0a.120bc.7f1d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88250:a129d217cb0f
Date: 2016-11-08 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/a129d217cb0f/

Log:	Win32: pass/skip a few tests

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1157,12 +1157,19 @@
     def setup_class(cls):
         cls.w_path = space.wrap(str(path))
 
-    def test_environ(self):
-        import posix
-        assert posix.environ['PATH']
-        del posix.environ['PATH']
-        def fn(): posix.environ['PATH']
-        raises(KeyError, fn)
+    if sys.platform != 'win32':
+        def test_environ(self):
+            import posix
+            assert posix.environ['PATH']
+            del posix.environ['PATH']
+            def fn(): posix.environ['PATH']
+            raises(KeyError, fn)
+    else:
+        def test_environ(self):
+            import nt
+            assert 'ADLDJSSLDFKJSD' not in nt.environ
+            def fn(): nt.environ['ADLDJSSLDFKJSD']
+            raises(KeyError, fn)
 
     if hasattr(__import__(os.name), "unsetenv"):
         def test_unsetenv_nonexisting(self):
@@ -1184,6 +1191,8 @@
 
 class AppTestPosixUnicode:
     def setup_class(cls):
+        if sys.platform == 'win32':
+            py.test.skip("Posix-only tests")
         if cls.runappdirect:
             # Can't change encoding
             try:

From pypy.commits at gmail.com  Wed Nov  9 03:50:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 00:50:46 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Win32: remove
 rposix.fdlistdir()
Message-ID: <5822e366.c24bc20a.3a44a.35e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88253:67c58d8dae1a
Date: 2016-11-08 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/67c58d8dae1a/

Log:	Win32: remove rposix.fdlistdir()

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -679,21 +679,22 @@
         raise OSError(error, "readdir failed")
     return result
 
-def fdlistdir(dirfd):
-    """
-    Like listdir(), except that the directory is specified as an open
-    file descriptor.
+if not _WIN32:
+    def fdlistdir(dirfd):
+        """
+        Like listdir(), except that the directory is specified as an open
+        file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.  To emulate the
-    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
-    file descriptor.
-    """
-    dirp = c_fdopendir(dirfd)
-    if not dirp:
-        error = get_saved_errno()
-        c_close(dirfd)
-        raise OSError(error, "opendir failed")
-    return _listdir(dirp, rewind=True)
+        Note: fdlistdir() closes the file descriptor.  To emulate the
+        Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+        file descriptor.
+        """
+        dirp = c_fdopendir(dirfd)
+        if not dirp:
+            error = get_saved_errno()
+            c_close(dirfd)
+            raise OSError(error, "opendir failed")
+        return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)

From pypy.commits at gmail.com  Wed Nov  9 03:50:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 00:50:48 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Win32: fix making sockets
 non-inheritable
Message-ID: <5822e368.6737c20a.fe636.f587@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88254:536a00a9e525
Date: 2016-11-08 20:02 +0100
http://bitbucket.org/pypy/pypy/changeset/536a00a9e525/

Log:	Win32: fix making sockets non-inheritable

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -641,6 +641,7 @@
 
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
+        # XXX 'traverse' is ignored
         win32traits = make_win32_traits(traits)
         with lltype.scoped_alloc(
                 win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data:
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1062,11 +1062,32 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
-def sock_set_inheritable(fd, inheritable):
-    try:
-        rposix.set_inheritable(fd, inheritable)
-    except OSError as e:
-        raise CSocketError(e.errno)
+if _c.WIN32:
+    def sock_set_inheritable(fd, inheritable):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            rwin32.set_handle_inheritable(handle, inheritable)
+        except WindowsError:
+            raise RSocketError("SetHandleInformation failed")   # xxx
+
+    def sock_get_inheritable(fd):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            return rwin32.get_handle_inheritable(handle)
+        except WindowsError:
+            raise RSocketError("GetHandleInformation failed")   # xxx
+else:
+    def sock_set_inheritable(fd, inheritable):
+        try:
+            rposix.set_inheritable(fd, inheritable)
+        except OSError as e:
+            raise CSocketError(e.errno)
+
+    def sock_get_inheritable(fd):
+        try:
+            return rposix.get_inheritable(fd)
+        except OSError as e:
+            raise CSocketError(e.errno)
 
 class SocketError(Exception):
     applevelerrcls = 'error'
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -481,6 +481,10 @@
 
     def set_inheritable(fd, inheritable):
         handle = get_osfhandle(fd)
+        set_handle_inheritable(handle, inheritable)
+
+    def set_handle_inheritable(handle, inheritable):
+        assert lltype.typeOf(handle) is HANDLE
         if inheritable:
             flags = HANDLE_FLAG_INHERIT
         else:
@@ -490,6 +494,10 @@
 
     def get_inheritable(fd):
         handle = get_osfhandle(fd)
+        return get_handle_inheritable(handle)
+
+    def get_handle_inheritable(handle):
+        assert lltype.typeOf(handle) is HANDLE
         pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
         try:
             if not _GetHandleInformation(handle, pflags):
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -124,8 +124,8 @@
         py.test.skip('No socketpair on Windows')
     for inh in [False, True]:
         s1, s2 = socketpair(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
-        assert rposix.get_inheritable(s2.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s2.fd) == inh
         s1.close()
         s2.close()
 
@@ -391,7 +391,7 @@
 def test_inheritable():
     for inh in [False, True]:
         s1 = RSocket(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
         s1.close()
 
 def test_getaddrinfo_http():

From pypy.commits at gmail.com  Wed Nov  9 03:50:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 00:50:43 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Win32: fix set_inheritable
 tests
Message-ID: <5822e363.c4811c0a.655bc.7f17@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88251:c03abff8ff75
Date: 2016-11-08 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/c03abff8ff75/

Log:	Win32: fix set_inheritable tests

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1166,6 +1166,10 @@
 @replace_os_function('pipe')
 def pipe(flags=0):
     # 'flags' might be ignored.  Check the result.
+    # The handles returned are always inheritable on Posix.
+    # The situation on Windows is not completely clear: I think
+    # it should always return non-inheritable handles, but CPython
+    # uses SECURITY_ATTRIBUTES to ensure that and we don't.
     if _WIN32:
         # 'flags' ignored
         ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
@@ -2146,8 +2150,9 @@
         handle_posix_error('mknodat', error)
 
 
-eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=[r"""
+if not _WIN32:
+    eci_inheritable = eci.merge(ExternalCompilationInfo(
+        separate_module_sources=[r"""
 #include <errno.h>
 #include <sys/ioctl.h>
 
@@ -2200,10 +2205,6 @@
 RPY_EXTERN
 int rpy_dup_noninheritable(int fd)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUPFD_CLOEXEC
     return fcntl(fd, F_DUPFD_CLOEXEC, 0);
 #else
@@ -2221,10 +2222,6 @@
 RPY_EXTERN
 int rpy_dup2_noninheritable(int fd, int fd2)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUP2FD_CLOEXEC
     return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
 
@@ -2249,33 +2246,41 @@
     return 0;
 #endif
 }
-    """ % {'HAVE_DUP3': HAVE_DUP3}],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
-                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
+        """ % {'HAVE_DUP3': HAVE_DUP3}],
+        post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                           'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n'
+                           ]))
 
-c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
-                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                compilation_info=eci_inheritable)
-c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
-                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                 compilation_info=eci_inheritable)
+    _c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    _c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                    rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                    compilation_info=eci_inheritable)
+    c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                     rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                     compilation_info=eci_inheritable)
 
-def set_inheritable(fd, inheritable):
-    result = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', result)
+    def set_inheritable(fd, inheritable):
+        result = _c_set_inheritable(fd, inheritable)
+        handle_posix_error('set_inheritable', result)
 
-def get_inheritable(fd):
-    res = c_get_inheritable(fd)
-    res = handle_posix_error('get_inheritable', res)
-    return res != 0
+    def get_inheritable(fd):
+        res = _c_get_inheritable(fd)
+        res = handle_posix_error('get_inheritable', res)
+        return res != 0
+
+else:
+    # _WIN32
+    from rpython.rlib.rwin32 import set_inheritable, get_inheritable
+    from rpython.rlib.rwin32 import c_dup_noninheritable
+    from rpython.rlib.rwin32 import c_dup2_noninheritable
+
 
 class SetNonInheritableCache(object):
     """Make one prebuilt instance of this for each path that creates
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -112,7 +112,7 @@
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
-                       STD_ERROR_HANDLE
+                       STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -473,3 +473,56 @@
     CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
     GetConsoleScreenBufferInfo = winexternal(
         "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
+
+    _GetHandleInformation = winexternal(
+        'GetHandleInformation', [HANDLE, LPDWORD], BOOL)
+    _SetHandleInformation = winexternal(
+        'SetHandleInformation', [HANDLE, DWORD, DWORD], BOOL)
+
+    def set_inheritable(fd, inheritable):
+        handle = get_osfhandle(fd)
+        if inheritable:
+            flags = HANDLE_FLAG_INHERIT
+        else:
+            flags = 0
+        if not _SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags):
+            raise lastSavedWindowsError("SetHandleInformation")
+
+    def get_inheritable(fd):
+        handle = get_osfhandle(fd)
+        pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
+        try:
+            if not _GetHandleInformation(handle, pflags):
+                raise lastSavedWindowsError("GetHandleInformation")
+            flags = pflags[0]
+        finally:
+            lltype.free(pflags, flavor='raw')
+        return (flags & HANDLE_FLAG_INHERIT) != 0
+
+    _GetFileType = winexternal('GetFileType', [HANDLE], DWORD)
+
+    def c_dup_noninheritable(fd1):
+        from rpython.rlib.rposix import c_dup
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        fd2 = c_dup(fd1)     # the inheritable version
+        if fd2 >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return fd2
+
+    def c_dup2_noninheritable(fd1, fd2):
+        from rpython.rlib.rposix import c_dup2
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        res = c_dup2(fd1, fd2)     # the inheritable version
+        if res >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return res
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -604,6 +604,9 @@
 def test_SetNonInheritableCache():
     cache = rposix.SetNonInheritableCache()
     fd1, fd2 = os.pipe()
+    if sys.platform == 'win32':
+        rposix.set_inheritable(fd1, True)
+        rposix.set_inheritable(fd2, True)
     assert rposix.get_inheritable(fd1) == True
     assert rposix.get_inheritable(fd1) == True
     assert cache.cached_inheritable == -1
@@ -616,6 +619,24 @@
     os.close(fd1)
     os.close(fd2)
 
+def test_dup_dup2_non_inheritable():
+    for preset in [False, True]:
+        fd1, fd2 = os.pipe()
+        rposix.set_inheritable(fd1, preset)
+        rposix.set_inheritable(fd2, preset)
+        fd3 = rposix.dup(fd1, True)
+        assert rposix.get_inheritable(fd3) == True
+        fd4 = rposix.dup(fd1, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd4, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd3, True)
+        assert rposix.get_inheritable(fd3) == True
+        os.close(fd1)
+        os.close(fd2)
+        os.close(fd3)
+        os.close(fd4)
+
 def test_sync():
     if sys.platform != 'win32':
         rposix.sync()

From pypy.commits at gmail.com  Wed Nov  9 03:50:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 00:50:45 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Win32: pass
 test_rposix.test_putenv
Message-ID: <5822e365.4a0e1c0a.2bb0.db79@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88252:b967d45102c1
Date: 2016-11-08 19:09 +0100
http://bitbucket.org/pypy/pypy/changeset/b967d45102c1/

Log:	Win32: pass test_rposix.test_putenv

diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -201,6 +201,8 @@
     # default implementation for platforms without a real unsetenv()
     r_putenv(name, '')
 
+REAL_UNSETENV = False
+
 if hasattr(__import__(os.name), 'unsetenv'):
     os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], rffi.INT,
                                   save_err=rffi.RFFI_SAVE_ERRNO)
@@ -222,3 +224,4 @@
     register_external(r_unsetenv, [str0], annmodel.s_None,
                       export_name='ll_os.ll_os_unsetenv',
                       llimpl=unsetenv_llimpl)
+    REAL_UNSETENV = True
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -470,11 +470,18 @@
         assert rposix.is_valid_fd(fd) == 0
 
     def test_putenv(self):
+        from rpython.rlib import rposix_environ
+
         def f():
             rposix.putenv(self.path, self.path)
             rposix.unsetenv(self.path)
 
-        interpret(f, []) # does not crash
+        interpret(f, [],     # does not crash
+                  malloc_check=rposix_environ.REAL_UNSETENV)
+        # If we have a real unsetenv(), check that it frees the string
+        # kept alive by putenv().  Otherwise, we can't check that,
+        # because unsetenv() will keep another string alive itself.
+    test_putenv.dont_track_allocations = True
 
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):

From pypy.commits at gmail.com  Wed Nov  9 06:05:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 03:05:54 -0800 (PST)
Subject: [pypy-commit] pypy default: test wrote just after the bounds of the
 array (test was not broke, just did to much work)
Message-ID: <58230312.e39cc20a.e2b77.6349@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88255:56484fd5b3d5
Date: 2016-11-09 12:05 +0100
http://bitbucket.org/pypy/pypy/changeset/56484fd5b3d5/

Log:	test wrote just after the bounds of the array (test was not broke,
	just did to much work)

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -865,7 +865,7 @@
         def f(bytecount, va, vb, vc):
             i = 0
             j = 0
-            while i < bytecount:
+            while j < bytecount:
                 myjitdriver.jit_merge_point()
                 a = raw_storage_getitem(rffi.DOUBLE,va,j)
                 b = raw_storage_getitem(rffi.INT,vb,i)

From pypy.commits at gmail.com  Wed Nov  9 06:13:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 03:13:58 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: account for vector guards
 while parsing the textual jit log
Message-ID: <582304f6.88571c0a.19783.1a18@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88256:8629fbc03fd8
Date: 2016-11-08 20:13 +0100
http://bitbucket.org/pypy/pypy/changeset/8629fbc03fd8/

Log:	account for vector guards while parsing the textual jit log

diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -62,7 +62,7 @@
         guard_dict = {}
         for loop_no, loop in enumerate(loops):
             for op in loop.operations:
-                if op.name.startswith('guard_'):
+                if op.name.startswith('guard_') or op.name.startswith('vec_guard_'):
                     guard_dict[int(op.descr[len('<Guard0x'):-1], 16)] = (op, loop)
         for loop in loops:
             if loop.comment:

From pypy.commits at gmail.com  Wed Nov  9 06:14:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 03:14:00 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: test wrote just after the
 bounds of the array (test was not broke, just did to much work)
Message-ID: <582304f8.45ad1c0a.ae9d6.1c56@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88257:bf1730c272d9
Date: 2016-11-09 12:05 +0100
http://bitbucket.org/pypy/pypy/changeset/bf1730c272d9/

Log:	test wrote just after the bounds of the array (test was not broke,
	just did to much work)

diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -867,7 +867,7 @@
         def f(bytecount, va, vb, vc):
             i = 0
             j = 0
-            while i < bytecount:
+            while j < bytecount:
                 myjitdriver.jit_merge_point()
                 a = raw_storage_getitem(rffi.DOUBLE,va,j)
                 b = raw_storage_getitem(rffi.INT,vb,i)

From pypy.commits at gmail.com  Wed Nov  9 07:37:54 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:37:54 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _rawffi
Message-ID: <582318a2.6a5cc20a.7e235.e1a0@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88258:6d72a3ca1108
Date: 2016-11-07 18:40 +0100
http://bitbucket.org/pypy/pypy/changeset/6d72a3ca1108/

Log:	_rawffi

diff --git a/pypy/module/_rawffi/alt/interp_ffitype.py b/pypy/module/_rawffi/alt/interp_ffitype.py
--- a/pypy/module/_rawffi/alt/interp_ffitype.py
+++ b/pypy/module/_rawffi/alt/interp_ffitype.py
@@ -37,7 +37,7 @@
 
     def descr_sizeof(self, space):
         try:
-            return space.wrap(self.sizeof())
+            return space.newint(self.sizeof())
         except ValueError:
             raise oefmt(space.w_ValueError,
                         "Operation not permitted on an incomplete type")
@@ -49,7 +49,7 @@
         return intmask(self.get_ffitype().c_alignment)
 
     def repr(self, space):
-        return space.wrap(self.__repr__())
+        return space.newtext(self.__repr__())
 
     def __repr__(self):
         name = self.name
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -114,7 +114,7 @@
         try:
             return func_caller.do_and_wrap(self.w_restype)
         except StackCheckError as e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
+            raise OperationError(space.w_ValueError, space.newtext(e.message))
         #return self._do_call(space, argchain)
 
     def free_temp_buffers(self, space):
@@ -129,7 +129,7 @@
         """
         Return the physical address in memory of the function
         """
-        return space.wrap(rffi.cast(rffi.LONG, self.func.funcsym))
+        return space.newint(rffi.cast(rffi.LONG, self.func.funcsym))
 
 
 class PushArgumentConverter(FromAppLevelConverter):
@@ -213,7 +213,7 @@
         # the correct value, and to be sure to handle the signed/unsigned case
         # correctly, we need to cast the result to the correct type.  After
         # that, we cast it back to LONG, because this is what we want to pass
-        # to space.wrap in order to get a nice applevel <int>.
+        # to space.newint in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
         call = self.func.call
@@ -337,11 +337,11 @@
         except KeyError:
             raise oefmt(space.w_ValueError,
                         "No symbol %s found in library %s", name, self.name)
-        return space.wrap(address_as_uint)
+        return space.newint(address_as_uint)
 
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
-    return space.wrap(W_CDLL(space, name, mode))
+    return W_CDLL(space, name, mode)
 
 
 W_CDLL.typedef = TypeDef(
@@ -358,7 +358,7 @@
 
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_windll(space, w_type, name, mode=-1):
-    return space.wrap(W_WinDLL(space, name, mode))
+    return W_WinDLL(space, name, mode)
 
 
 W_WinDLL.typedef = TypeDef(
@@ -372,7 +372,7 @@
 
 def get_libc(space):
     try:
-        return space.wrap(W_CDLL(space, get_libc_name(), -1))
+        return W_CDLL(space, get_libc_name(), -1)
     except OSError as e:
         raise wrap_oserror(space, e)
 
diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py
--- a/pypy/module/_rawffi/alt/interp_struct.py
+++ b/pypy/module/_rawffi/alt/interp_struct.py
@@ -176,7 +176,7 @@
 
     def getaddr(self, space):
         addr = rffi.cast(rffi.ULONG, self.rawmem)
-        return space.wrap(addr)
+        return space.newint(addr)
 
     @unwrap_spec(name=str)
     def getfield(self, space, name):
diff --git a/pypy/module/_rawffi/alt/type_converter.py b/pypy/module/_rawffi/alt/type_converter.py
--- a/pypy/module/_rawffi/alt/type_converter.py
+++ b/pypy/module/_rawffi/alt/type_converter.py
@@ -202,7 +202,7 @@
             return self._longlong(w_ffitype)
         elif w_ffitype.is_signed():
             intval = self.get_signed(w_ffitype)
-            return space.wrap(intval)
+            return space.newint(intval)
         elif (w_ffitype is app_types.ulonglong or
               w_ffitype is app_types.ulong or (libffi.IS_32_BIT and
                                                w_ffitype is app_types.uint)):
@@ -216,19 +216,19 @@
             # and app-evel <long>.  This is why we need to treat it separately
             # than the other unsigned types.
             uintval = self.get_unsigned(w_ffitype)
-            return space.wrap(uintval)
+            return space.newint(uintval)
         elif w_ffitype.is_unsigned(): # note that ulong is handled just before
             intval = self.get_unsigned_which_fits_into_a_signed(w_ffitype)
-            return space.wrap(intval)
+            return space.newint(intval)
         elif w_ffitype.is_pointer():
             uintval = self.get_pointer(w_ffitype)
-            return space.wrap(uintval)
+            return space.newint(uintval)
         elif w_ffitype.is_char():
             ucharval = self.get_char(w_ffitype)
-            return space.wrap(chr(ucharval))
+            return space.newbytes(chr(ucharval))
         elif w_ffitype.is_unichar():
             wcharval = self.get_unichar(w_ffitype)
-            return space.wrap(unichr(wcharval))
+            return space.newunicode(unichr(wcharval))
         elif w_ffitype.is_double():
             return self._float(w_ffitype)
         elif w_ffitype.is_singlefloat():
@@ -253,10 +253,10 @@
         # depending on whether longlongs are supported
         if w_ffitype is app_types.slonglong:
             longlongval = self.get_longlong(w_ffitype)
-            return self.space.wrap(longlongval)
+            return self.space.newint(longlongval)
         elif w_ffitype is app_types.ulonglong:
             ulonglongval = self.get_ulonglong(w_ffitype)
-            return self.space.wrap(ulonglongval)
+            return self.space.newint(ulonglongval)
         else:
             self.error(w_ffitype)
 
@@ -264,13 +264,13 @@
         # a separate function, which can be seen by the jit or not,
         # depending on whether floats are supported
         floatval = self.get_float(w_ffitype)
-        return self.space.wrap(floatval)
+        return self.space.newfloat(floatval)
 
     def _singlefloat(self, w_ffitype):
         # a separate function, which can be seen by the jit or not,
         # depending on whether singlefloats are supported
         singlefloatval = self.get_singlefloat(w_ffitype)
-        return self.space.wrap(float(singlefloatval))
+        return self.space.newfloat(float(singlefloatval))
 
     def error(self, w_ffitype):
         raise oefmt(self.space.w_TypeError,
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -49,16 +49,16 @@
                 w_item = items_w[num]
                 unwrap_value(space, write_ptr, result.ll_buffer, num,
                              self.itemcode, w_item)
-        return space.wrap(result)
+        return result
 
     def descr_repr(self, space):
-        return space.wrap("<_rawffi.Array '%s' (%d, %d)>" % (self.itemcode,
-                                                             self.size,
-                                                             self.alignment))
+        return space.newtext("<_rawffi.Array '%s' (%d, %d)>" % (self.itemcode,
+                                                                self.size,
+                                                                self.alignment))
 
     @unwrap_spec(address=r_uint, length=int)
     def fromaddress(self, space, address, length):
-        return space.wrap(W_ArrayInstance(space, self, length, address))
+        return W_ArrayInstance(space, self, length, address)
 
 PRIMITIVE_ARRAY_TYPES = {}
 for _code in TYPEMAP:
@@ -94,8 +94,8 @@
 
     def descr_repr(self, space):
         addr = rffi.cast(lltype.Unsigned, self.ll_buffer)
-        return space.wrap("<_rawffi array %x of length %d>" % (addr,
-                                                               self.length))
+        return space.newtext("<_rawffi array %x of length %d>" % (addr,
+                                                                  self.length))
 
     # This only allows non-negative indexes.  Arrays of shape 'c' also
     # support simple slices.
@@ -137,13 +137,13 @@
             return self.getitem(space, num)
 
     def getlength(self, space):
-        return space.wrap(self.length)
+        return space.newint(self.length)
 
     @unwrap_spec(num=int)
     def descr_itemaddress(self, space, num):
         itemsize = self.shape.size
         ptr = rffi.ptradd(self.ll_buffer, itemsize * num)
-        return space.wrap(rffi.cast(lltype.Unsigned, ptr))
+        return space.newint(rffi.cast(lltype.Unsigned, ptr))
 
     def getrawsize(self):
         itemsize = self.shape.size
@@ -155,9 +155,9 @@
         letter = self.shape.itemcode
         if letter != 'c':
             raise oefmt(space.w_TypeError, "only 'c' arrays support slicing")
-        w_start = space.getattr(w_slice, space.wrap('start'))
-        w_stop = space.getattr(w_slice, space.wrap('stop'))
-        w_step = space.getattr(w_slice, space.wrap('step'))
+        w_start = space.getattr(w_slice, space.newtext('start'))
+        w_stop = space.getattr(w_slice, space.newtext('stop'))
+        w_step = space.getattr(w_slice, space.newtext('step'))
 
         if space.is_w(w_start, space.w_None):
             start = 0
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -45,7 +45,7 @@
                     space, rffi.cast(rffi.SIZE_T, ll_args[i]))
             else:
                 # XXX other types?
-                args_w[i] = space.wrap(rffi.cast(rffi.ULONG, ll_args[i]))
+                args_w[i] = space.newint(rffi.cast(rffi.ULONG, ll_args[i]))
         w_res = space.call(w_callable, space.newtuple(args_w))
         if callback_ptr.result is not None: # don't return void
             ptr = ll_res
@@ -60,8 +60,8 @@
                         break
             unwrap_value(space, write_ptr, ptr, 0, letter, w_res)
     except OperationError as e:
-        tbprint(space, space.wrap(e.get_traceback()),
-                space.wrap(e.errorstr(space)))
+        tbprint(space, e.get_traceback(),
+                space.newtext(e.errorstr(space)))
         # force the result to be zero
         if callback_ptr.result is not None:
             resshape = letter2tp(space, callback_ptr.result)
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -163,10 +163,13 @@
         and restype
         """
         resshape = unpack_resshape(space, w_restype)
-        w = space.wrap
+        if resshape is None:
+            w_resshape = space.w_None
+        else:
+            w_resshape = resshape
         argtypes_w = space.fixedview(w_argtypes)
         w_argtypes = space.newtuple(argtypes_w)
-        w_key = space.newtuple([w_name, w_argtypes, w(resshape)])
+        w_key = space.newtuple([w_name, w_argtypes, w_resshape])
         try:
             return space.getitem(self.w_cache, w_key)
         except OperationError as e:
@@ -222,7 +225,7 @@
                                         self.cdll.getaddressindll(name))
         except KeyError:
             raise oefmt(space.w_ValueError, "Cannot find symbol %s", name)
-        return space.wrap(address_as_uint)
+        return space.newint(address_as_uint)
 
 @unwrap_spec(name='str_or_None')
 def descr_new_cdll(space, w_type, name):
@@ -232,7 +235,7 @@
         raise wrap_dlopenerror(space, e, name)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(W_CDLL(space, name, cdll))
+    return W_CDLL(space, name, cdll)
 
 W_CDLL.typedef = TypeDef(
     'CDLL',
@@ -301,8 +304,8 @@
 
 def segfault_exception(space, reason):
     w_mod = space.getbuiltinmodule("_rawffi")
-    w_exception = space.getattr(w_mod, space.wrap("SegfaultException"))
-    return OperationError(w_exception, space.wrap(reason))
+    w_exception = space.getattr(w_mod, space.newtext("SegfaultException"))
+    return OperationError(w_exception, space.newtext(reason))
 
 class W_DataShape(W_Root):
     _array_shapes = None
@@ -322,8 +325,8 @@
 
     @unwrap_spec(n=int)
     def descr_size_alignment(self, space, n=1):
-        return space.newtuple([space.wrap(self.size * n),
-                               space.wrap(self.alignment)])
+        return space.newtuple([space.newint(self.size * n),
+                               space.newint(self.alignment)])
 
 
 class W_DataInstance(W_Root):
@@ -339,7 +342,7 @@
         self._ll_buffer = self.ll_buffer
 
     def getbuffer(self, space):
-        return space.wrap(rffi.cast(lltype.Unsigned, self.ll_buffer))
+        return space.newint(rffi.cast(lltype.Unsigned, self.ll_buffer))
 
     def buffer_advance(self, n):
         self.ll_buffer = rffi.ptradd(self.ll_buffer, n)
@@ -347,8 +350,8 @@
     def byptr(self, space):
         from pypy.module._rawffi.array import ARRAY_OF_PTRS
         array = ARRAY_OF_PTRS.allocate(space, 1)
-        array.setitem(space, 0, space.wrap(self))
-        return space.wrap(array)
+        array.setitem(space, 0, self)
+        return array
 
     def free(self, space):
         if not self._ll_buffer:
@@ -433,10 +436,13 @@
         if letter == c:
             if c in TYPEMAP_PTR_LETTERS:
                 res = func(add_arg, argdesc, rffi.VOIDP)
-                return space.wrap(rffi.cast(lltype.Unsigned, res))
+                return space.newint(rffi.cast(lltype.Unsigned, res))
+            elif c == 'c':
+                return space.newbytes(func(add_arg, argdesc, ll_type))
             elif c == 'f' or c == 'd' or c == 'g':
-                return space.wrap(float(func(add_arg, argdesc, ll_type)))
+                return space.newfloat(float(func(add_arg, argdesc, ll_type)))
             else:
+                # YYY hard
                 return space.wrap(func(add_arg, argdesc, ll_type))
     raise oefmt(space.w_TypeError, "cannot directly read value")
 
@@ -455,7 +461,7 @@
             self.narrow_integer = is_narrow_integer_type(resshape.itemcode.lower())
 
     def getbuffer(self, space):
-        return space.wrap(rffi.cast(lltype.Unsigned, self.ptr.funcsym))
+        return space.newint(rffi.cast(lltype.Unsigned, self.ptr.funcsym))
 
     def byptr(self, space):
         from pypy.module._rawffi.array import ARRAY_OF_PTRS
@@ -465,7 +471,7 @@
             # XXX this is needed, because functions tend to live forever
             #     hence our testing is not performing that well
             del tracker.alloced[rffi.cast(lltype.Signed, array.ll_buffer)]
-        return space.wrap(array)
+        return array
 
     def call(self, space, args_w):
         from pypy.module._rawffi.array import W_ArrayInstance
@@ -516,12 +522,12 @@
                     # we get a 8 byte value in big endian
                     n = rffi.sizeof(lltype.Signed) - result.shape.size
                     result.buffer_advance(n)
-                return space.wrap(result)
+                return result
             else:
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
                 return space.w_None
         except StackCheckError as e:
-            raise OperationError(space.w_ValueError, space.wrap(e.message))
+            raise OperationError(space.w_ValueError, space.newtext(e.message))
 
 @unwrap_spec(addr=r_uint, flags=int)
 def descr_new_funcptr(space, w_tp, addr, w_args, w_res, flags=FUNCFLAG_CDECL):
@@ -537,7 +543,7 @@
                          flags)
     except LibFFIError:
         raise got_libffi_error(space)
-    return space.wrap(W_FuncPtr(space, ptr, argshapes, resshape))
+    return W_FuncPtr(space, ptr, argshapes, resshape)
 
 W_FuncPtr.typedef = TypeDef(
     'FuncPtr',
@@ -555,7 +561,7 @@
             raise oefmt(space.w_ValueError, "Expecting string of length one")
         tp_letter = tp_letter[0] # fool annotator
         try:
-            return space.wrap(intmask(getattr(TYPEMAP[tp_letter], name)))
+            return space.newint(intmask(getattr(TYPEMAP[tp_letter], name)))
         except KeyError:
             raise oefmt(space.w_ValueError, "Unknown type specification %s",
                         tp_letter)
@@ -584,7 +590,7 @@
         s = rffi.wcharp2unicode(wcharp_addr)
     else:
         s = rffi.wcharp2unicoden(wcharp_addr, maxlength)
-    return space.wrap(s)
+    return space.newunicode(s)
 
 @unwrap_spec(address=r_uint, maxlength=int)
 def charp2rawstring(space, address, maxlength=-1):
@@ -598,7 +604,7 @@
     if maxlength == -1:
         return wcharp2unicode(space, address)
     s = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, address), maxlength)
-    return space.wrap(s)
+    return space.newunicode(s)
 
 @unwrap_spec(address=r_uint, newcontent='bufferstr')
 def rawstring2charp(space, address, newcontent):
@@ -615,8 +621,8 @@
     @unwrap_spec(hresult=int)
     def check_HRESULT(space, hresult):
         if rwin32.FAILED(hresult):
-            raise OperationError(space.w_WindowsError, space.wrap(hresult))
-        return space.wrap(hresult)
+            raise OperationError(space.w_WindowsError, space.newint(hresult))
+        return space.newint(hresult)
 
 def get_libc(space):
     name = get_libc_name()
@@ -624,10 +630,10 @@
         cdll = CDLL(name)
     except OSError as e:
         raise wrap_oserror(space, e)
-    return space.wrap(W_CDLL(space, name, cdll))
+    return W_CDLL(space, name, cdll)
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_alterrno())
+    return space.newint(rposix.get_saved_alterrno())
 
 def set_errno(space, w_errno):
     rposix.set_saved_alterrno(space.int_w(w_errno))
@@ -644,7 +650,7 @@
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
     def get_last_error(space):
-        return space.wrap(0)
+        return space.newint(0)
     @unwrap_spec(error=int)
     def set_last_error(space, error):
         pass
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -192,30 +192,30 @@
 
     @unwrap_spec(autofree=bool)
     def descr_call(self, space, autofree=False):
-        return space.wrap(self.allocate(space, 1, autofree))
+        return self.allocate(space, 1, autofree)
 
     def descr_repr(self, space):
         fieldnames = ' '.join(["'%s'" % name for name, _, _ in self.fields])
-        return space.wrap("<_rawffi.Structure %s (%d, %d)>" % (fieldnames,
-                                                               self.size,
-                                                               self.alignment))
+        return space.newtext("<_rawffi.Structure %s (%d, %d)>" % (fieldnames,
+                                                                  self.size,
+                                                                  self.alignment))
 
     @unwrap_spec(address=r_uint)
     def fromaddress(self, space, address):
-        return space.wrap(W_StructureInstance(space, self, address))
+        return W_StructureInstance(space, self, address)
 
     @unwrap_spec(attr=str)
     def descr_fieldoffset(self, space, attr):
         index = self.getindex(space, attr)
-        return space.wrap(self.ll_positions[index])
+        return space.newint(self.ll_positions[index])
 
     @unwrap_spec(attr=str)
     def descr_fieldsize(self, space, attr):
         index = self.getindex(space, attr)
         if self.ll_bitsizes and index < len(self.ll_bitsizes):
-            return space.wrap(self.ll_bitsizes[index])
+            return space.newint(self.ll_bitsizes[index])
         else:
-            return space.wrap(self.fields[index][1].size)
+            return space.newint(self.fields[index][1].size)
 
     # get the corresponding ffi_type
     ffi_struct = lltype.nullptr(clibffi.FFI_STRUCT_P.TO)
@@ -262,7 +262,7 @@
     else:
         fields = unpack_fields(space, w_shapeinfo)
         S = W_Structure(space, fields, 0, 0, union, pack)
-    return space.wrap(S)
+    return S
 
 W_Structure.typedef = TypeDef(
     'Structure',
@@ -347,7 +347,7 @@
 
     def descr_repr(self, space):
         addr = rffi.cast(lltype.Unsigned, self.ll_buffer)
-        return space.wrap("<_rawffi struct %x>" % (addr,))
+        return space.newtext("<_rawffi struct %x>" % (addr,))
 
     @unwrap_spec(attr=str)
     def getattr(self, space, attr):
@@ -369,7 +369,7 @@
     def descr_fieldaddress(self, space, attr):
         i = self.shape.getindex(space, attr)
         ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i])
-        return space.wrap(rffi.cast(lltype.Unsigned, ptr))
+        return space.newint(rffi.cast(lltype.Unsigned, ptr))
 
     def getrawsize(self):
         return self.shape.size
diff --git a/pypy/module/_rawffi/tracker.py b/pypy/module/_rawffi/tracker.py
--- a/pypy/module/_rawffi/tracker.py
+++ b/pypy/module/_rawffi/tracker.py
@@ -24,7 +24,7 @@
     if not tracker.DO_TRACING:
         raise oefmt(space.w_RuntimeError,
                     "DO_TRACING not enabled in this PyPy")
-    return space.wrap(len(tracker.alloced))
+    return space.newint(len(tracker.alloced))
 
 def print_alloced_objects(space):
     xxx

From pypy.commits at gmail.com  Wed Nov  9 07:37:56 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:37:56 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: micronumpy
Message-ID: <582318a4.c24bc20a.3a44a.92fc@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88259:613414c4df6c
Date: 2016-11-08 12:52 +0100
http://bitbucket.org/pypy/pypy/changeset/613414c4df6c/

Log:	micronumpy

diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py
--- a/pypy/module/micronumpy/arrayops.py
+++ b/pypy/module/micronumpy/arrayops.py
@@ -105,13 +105,13 @@
         raise oefmt(space.w_ValueError, "need at least one array to concatenate")
     args_w = [convert_to_array(space, w_arg) for w_arg in args_w]
     if w_axis is None:
-        w_axis = space.wrap(0)
+        w_axis = space.newint(0)
     if space.is_none(w_axis):
         args_w = [w_arg.reshape(space,
                                 space.newlist([w_arg.descr_get_size(space)]),
                                 w_arg.get_order())
                   for w_arg in args_w]
-        w_axis = space.wrap(0)
+        w_axis = space.newint(0)
     dtype = args_w[0].get_dtype()
     shape = args_w[0].get_shape()[:]
     ndim = len(shape)
@@ -184,7 +184,7 @@
 
 
 def count_nonzero(space, w_obj):
-    return space.wrap(loop.count_all_true(convert_to_array(space, w_obj)))
+    return space.newint(loop.count_all_true(convert_to_array(space, w_obj)))
 
 
 def choose(space, w_arr, w_choices, w_out, w_mode):
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -136,7 +136,7 @@
         if w_val is not None:
             w_val = dtype.coerce(space, w_val)
         else:
-            w_val = dtype.coerce(space, space.wrap(0))
+            w_val = dtype.coerce(space, space.newint(0))
         return convert_to_array(space, w_val)
 
     @staticmethod
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -196,11 +196,11 @@
 
     def descr_str(self, space):
         tp = self.get_dtype(space).itemtype
-        return space.wrap(tp.str_format(self, add_quotes=False))
+        return space.newtext(tp.str_format(self, add_quotes=False))
 
     def descr_repr(self, space):
         tp = self.get_dtype(space).itemtype
-        return space.wrap(tp.str_format(self, add_quotes=True))
+        return space.newtext(tp.str_format(self, add_quotes=True))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -209,7 +209,7 @@
         return space.hash(self.item(space))
 
     def descr___array_priority__(self, space):
-        return space.wrap(0.0)
+        return space.newfloat(0.0)
 
     def descr_index(self, space):
         return space.index(self.item(space))
@@ -242,7 +242,7 @@
         return space.hex(self.descr_int(space))
 
     def descr_nonzero(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.bool(self))
+        return space.newbool(self.get_dtype(space).itemtype.bool(self))
 
     # TODO: support all kwargs in ufuncs like numpy ufunc_object.c
     sig = None
@@ -385,16 +385,16 @@
         return self.get_dtype(space)
 
     def descr_get_size(self, space):
-        return space.wrap(1)
+        return space.newint(1)
 
     def descr_get_itemsize(self, space):
-        return space.wrap(self.get_dtype(space).elsize)
+        return space.newint(self.get_dtype(space).elsize)
 
     def descr_get_shape(self, space):
         return space.newtuple([])
 
     def descr_get_ndim(self, space):
-        return space.wrap(0)
+        return space.newint(0)
 
     def descr_copy(self, space):
         return self.convert_to(space, self.get_dtype(space))
@@ -412,18 +412,18 @@
         return self.get_dtype(space).itemtype.byteswap(self)
 
     def descr_tostring(self, space, __args__):
-        w_meth = space.getattr(self.descr_ravel(space), space.wrap('tostring'))
+        w_meth = space.getattr(self.descr_ravel(space), space.newtext('tostring'))
         return space.call_args(w_meth, __args__)
 
     def descr_reshape(self, space, __args__):
-        w_meth = space.getattr(self.descr_ravel(space), space.wrap('reshape'))
+        w_meth = space.getattr(self.descr_ravel(space), space.newtext('reshape'))
         w_res = space.call_args(w_meth, __args__)
         if isinstance(w_res, W_NDimArray) and len(w_res.get_shape()) == 0:
             return w_res.get_scalar_value()
         return w_res
 
     def descr_nd_nonzero(self, space, __args__):
-        w_meth = space.getattr(self.descr_ravel(space), space.wrap('nonzero'))
+        w_meth = space.getattr(self.descr_ravel(space), space.newtext('nonzero'))
         return space.call_args(w_meth, __args__)
 
     def descr_get_real(self, space):
@@ -579,7 +579,7 @@
             read_val = dtype.read(self.arr, self.ofs, ofs)
         if isinstance (read_val, W_StringBox):
             # StringType returns a str
-            return space.wrap(dtype.itemtype.to_str(read_val))
+            return space.newbytes(dtype.itemtype.to_str(read_val))
         return read_val
 
     def descr_iter(self, space):
@@ -635,7 +635,7 @@
         if dtype.is_unicode():
             return self
         elif dtype.is_object():
-            return W_ObjectBox(space.wrap(self._value))
+            return W_ObjectBox(space.newunicode(self._value))
         else:
             raise oefmt(space.w_NotImplementedError,
                         "Conversion from unicode not implemented yet")
diff --git a/pypy/module/micronumpy/broadcast.py b/pypy/module/micronumpy/broadcast.py
--- a/pypy/module/micronumpy/broadcast.py
+++ b/pypy/module/micronumpy/broadcast.py
@@ -67,19 +67,19 @@
                             self.op_flags[i], self)
 
     def descr_iter(self, space):
-        return space.wrap(self)
+        return self
 
     def descr_get_shape(self, space):
-        return space.newtuple([space.wrap(i) for i in self.shape])
+        return space.newtuple([space.newint(i) for i in self.shape])
 
     def descr_get_size(self, space):
-        return space.wrap(self.size)
+        return space.newint(self.size)
 
     def descr_get_index(self, space):
-        return space.wrap(self.index)
+        return space.newint(self.index)
 
     def descr_get_numiter(self, space):
-        return space.wrap(len(self.iters))
+        return space.newint(len(self.iters))
 
     @jit.unroll_safe
     def descr_next(self, space):
diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -127,18 +127,18 @@
         raise oefmt(space.w_TypeError,
             "did not understand one of the types; 'None' not accepted")
     if isinstance(w_from, W_NDimArray):
-        return space.wrap(can_cast_array(space, w_from, target, casting))
+        return space.newbool(can_cast_array(space, w_from, target, casting))
     elif is_scalar_w(space, w_from):
         w_scalar = as_scalar(space, w_from)
         w_arr = W_NDimArray.from_scalar(space, w_scalar)
-        return space.wrap(can_cast_array(space, w_arr, target, casting))
+        return space.newbool(can_cast_array(space, w_arr, target, casting))
 
     try:
         origin = as_dtype(space, w_from, allow_None=False)
     except TypeError:
         raise oefmt(space.w_TypeError,
             "did not understand one of the types; 'None' not accepted")
-    return space.wrap(can_cast_type(space, origin, target, casting))
+    return space.newbool(can_cast_type(space, origin, target, casting))
 
 kind_ordering = {
     Bool.kind: 0, ULong.kind: 1, Long.kind: 2,
@@ -328,7 +328,7 @@
             space.int_w(w_obj)
         except OperationError as e:
             if e.match(space, space.w_OverflowError):
-                if space.is_true(space.le(w_obj, space.wrap(0))):
+                if space.is_true(space.le(w_obj, space.newint(0))):
                     return int64_dtype
                 return uint64_dtype
             raise
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -190,6 +190,10 @@
             return StringObject(obj)
         raise NotImplementedError
 
+    def newtext(self, obj):
+        return StringObject(obj)
+    newbytes = newtext
+
     def newlist(self, items):
         return ListObject(items)
 
@@ -197,7 +201,7 @@
         return ComplexObject(r, i)
 
     def newfloat(self, f):
-        return self.float(f)
+        return FloatObject(f)
 
     def newslice(self, start, stop, step):
         return SliceObject(self.int_w(start), self.int_w(stop),
@@ -293,6 +297,7 @@
         if isinstance(w_obj, StringObject):
             return w_obj.v
         raise NotImplementedError
+    text_w = str_w
 
     def unicode_w(self, w_obj):
         # XXX
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -233,7 +233,7 @@
         elif shape_len > 1:
             raise IndexError
         idx = support.index_w(space, w_idx)
-        return self._lookup_by_index(space, [space.wrap(idx)])
+        return self._lookup_by_index(space, [space.newint(idx)])
 
     @jit.unroll_safe
     def _prepare_slice_args(self, space, w_idx):
@@ -366,7 +366,7 @@
         w_res = W_NDimArray.from_shape(space, [s, nd], index_type)
         loop.nonzero(w_res, self, box)
         w_res = w_res.implementation.swapaxes(space, w_res, 0, 1)
-        l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)]
+        l_w = [w_res.descr_getitem(space, space.newint(d)) for d in range(nd)]
         return space.newtuple(l_w)
 
     ##def get_storage(self):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -45,10 +45,10 @@
 
 def try_interface_method(space, w_object, copy):
     try:
-        w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
+        w_interface = space.getattr(w_object, space.newtext("__array_interface__"))
         if w_interface is None:
             return None, False
-        version_w = space.finditem(w_interface, space.wrap("version"))
+        version_w = space.finditem(w_interface, space.newtext("version"))
         if version_w is None:
             raise oefmt(space.w_ValueError, "__array_interface__ found without"
                         " 'version' key")
@@ -60,11 +60,11 @@
             raise oefmt(space.w_ValueError,
                     "__array_interface__ version %d not supported", version)
         # make a view into the data
-        w_shape = space.finditem(w_interface, space.wrap('shape'))
-        w_dtype = space.finditem(w_interface, space.wrap('typestr'))
-        w_descr = space.finditem(w_interface, space.wrap('descr'))
-        w_data = space.finditem(w_interface, space.wrap('data'))
-        w_strides = space.finditem(w_interface, space.wrap('strides'))
+        w_shape = space.finditem(w_interface, space.newtext('shape'))
+        w_dtype = space.finditem(w_interface, space.newtext('typestr'))
+        w_descr = space.finditem(w_interface, space.newtext('descr'))
+        w_data = space.finditem(w_interface, space.newtext('data'))
+        w_strides = space.finditem(w_interface, space.newtext('strides'))
         if w_shape is None or w_dtype is None:
             raise oefmt(space.w_ValueError,
                     "__array_interface__ missing one or more required keys: shape, typestr"
@@ -96,7 +96,7 @@
                                 start=offset), read_only
         if w_data is None:
             w_data = w_object
-        w_offset = space.finditem(w_interface, space.wrap('offset'))
+        w_offset = space.finditem(w_interface, space.newtext('offset'))
         if w_offset is None:
             offset = 0
         else:
@@ -115,11 +115,11 @@
         raise
 
 def _descriptor_from_pep3118_format(space, c_format):
-    descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+    descr = descriptor.decode_w_dtype(space, space.newtext(c_format))
     if descr:
         return descr
     msg = "invalid PEP 3118 format string: '%s'" % c_format
-    space.warn(space.wrap(msg), space.w_RuntimeWarning)
+    space.warn(space.newtext(msg), space.w_RuntimeWarning)
     return None 
 
 def _array_from_buffer_3118(space, w_object, dtype):
@@ -142,7 +142,7 @@
         if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
             msg = ("Item size computed from the PEP 3118 buffer format "
                   "string does not match the actual item size.")
-            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            space.warn(space.newtext(msg), space.w_RuntimeWarning)
             return w_object
         dtype = descr 
     elif not dtype:
@@ -170,7 +170,7 @@
         elif nd > 1:
             msg = ("ndim computed from the PEP 3118 buffer format "
                    "is greater than 1, but shape is NULL.")
-            space.warn(space.wrap(msg), space.w_RuntimeWarning)
+            space.warn(space.newtext(msg), space.w_RuntimeWarning)
             return w_object
     try:
         w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
@@ -238,7 +238,7 @@
             must_copy |= (npy_order == NPY.CORDER and not flags & NPY.ARRAY_C_CONTIGUOUS)
             must_copy |= (npy_order == NPY.FORTRANORDER and not flags & NPY.ARRAY_F_CONTIGUOUS)
             if must_copy:
-                return w_object.descr_copy(space, space.wrap(npy_order))
+                return w_object.descr_copy(space, space.newint(npy_order))
             else:
                 return w_object
         if subok and not type(w_object) is W_NDimArray:
@@ -338,9 +338,9 @@
     from pypy.objspace.std.bufferobject import W_Buffer
     shape = [space.len_w(w_iterable)]
     if space.isinstance_w(w_iterable, space.w_buffer):
-        batch = [space.wrap(0)] * shape[0]
+        batch = [space.newint(0)] * shape[0]
         for i in range(shape[0]):
-            batch[i] = space.ord(space.getitem(w_iterable, space.wrap(i)))
+            batch[i] = space.ord(space.getitem(w_iterable, space.newint(i)))
     else:
         batch = space.listview(w_iterable)
     while True:
@@ -498,7 +498,7 @@
         ai.setitem(state, val)
         state = ai.next(state)
 
-    return space.wrap(a)
+    return a
 
 
 def _fromstring_bin(space, s, count, length, dtype):
@@ -516,7 +516,7 @@
 
     a = W_NDimArray.from_shape(space, [count], dtype=dtype)
     loop.fromstring_loop(space, a, dtype, itemsize, s)
-    return space.wrap(a)
+    return a
 
 
 @unwrap_spec(s=str, count=int, sep=str, w_dtype=WrappedDefault(None))
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -222,8 +222,8 @@
         else:
             name = self.get_name()
         if self.is_flexible() and self.elsize != 0:
-            return space.wrap(name + str(self.elsize * 8))
-        return space.wrap(name)
+            return space.newtext(name + str(self.elsize * 8))
+        return space.newtext(name)
 
     def descr_get_str(self, space, ignore='|', simple=True):
         if not simple and self.fields and len(self.fields) > 0:
@@ -233,10 +233,10 @@
             total += s
         if not simple and total > 0:
             return space.newtuple(
-                [space.wrap(self.subdtype.get_str(ignore='')), 
-                 space.newtuple([space.wrap(s) for s in self.shape]),
+                [space.newtext(self.subdtype.get_str(ignore='')), 
+                 space.newtuple([space.newint(s) for s in self.shape]),
                 ])
-        return space.wrap(self.get_str(ignore=ignore))
+        return space.newtext(self.get_str(ignore=ignore))
 
     def get_str(self, ignore='|'):
         basic = self.kind
@@ -257,7 +257,7 @@
         if style == 'descr':
             simple = True
         if not self.is_record():
-            return space.newlist([space.newtuple([space.wrap(""),
+            return space.newlist([space.newtuple([space.newtext(""),
                                                   self.descr_get_str(space, simple=simple)])])
         elif (self.alignment > 1 and not style.startswith('descr')) or force_dict:
             # we need to force a sorting order for the keys,
@@ -324,7 +324,7 @@
                 s_as_list += titles + [', ']
                     
             s_as_list += ["'itemsize':", str(self.elsize), suffix]
-            return space.wrap(''.join(s_as_list))
+            return space.newtext(''.join(s_as_list))
         else:
             descr = []
             total = 0
@@ -337,10 +337,10 @@
                 total += subdtype.elsize
                 ignore = '|'
                 if title:
-                    subdescr = [space.newtuple([space.wrap(title), space.wrap(name)])]
+                    subdescr = [space.newtuple([space.newtext(title), space.newtext(name)])]
                     ignore = ''
                 else:
-                    subdescr = [space.wrap(name)]
+                    subdescr = [space.newtext(name)]
                 if subdtype.is_record():
                     subdescr.append(subdtype.descr_get_descr(space, style))
                 elif subdtype.subdtype is not None:
@@ -351,19 +351,19 @@
                     subdescr.append(subdtype.descr_get_shape(space))
                 descr.append(space.newtuple(subdescr[:]))
             if self.alignment >= 0 and not style.endswith('subdtype'):
-                return space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
+                return space.newtext(space.str_w(space.repr(space.newlist(descr))) + ', align=True')                 
             return space.newlist(descr)
 
     def descr_get_hasobject(self, space):
-        return space.wrap(self.is_object())
+        return space.newbool(self.is_object())
 
     def descr_get_isbuiltin(self, space):
         if self.fields is None:
-            return space.wrap(1)
-        return space.wrap(0)
+            return space.newint(1)
+        return space.newint(0)
 
     def descr_get_isnative(self, space):
-        return space.wrap(self.is_native())
+        return space.newbool(self.is_native())
 
     def descr_get_base(self, space):
         return space.wrap(self.base)
@@ -375,10 +375,10 @@
                                self.descr_get_shape(space)])
 
     def descr_get_shape(self, space):
-        return space.newtuple([space.wrap(dim) for dim in self.shape])
+        return space.newtuple([space.newint(dim) for dim in self.shape])
 
     def descr_get_flags(self, space):
-        return space.wrap(self.flags)
+        return space.newint(self.flags)
 
     def descr_get_fields(self, space):
         if not self.fields:
@@ -387,18 +387,18 @@
         for name, title in self.names:
             offset, subdtype = self.fields[name]
             if title is not None:
-                w_nt = space.newtuple([space.wrap(name), space.wrap(title)]) 
+                w_nt = space.newtuple([space.newtext(name), space.newtext(title)]) 
                 space.setitem(w_fields, w_nt,
-                          space.newtuple([subdtype, space.wrap(offset)]))
+                          space.newtuple([subdtype, space.newint(offset)]))
             else:
-                space.setitem(w_fields, space.wrap(name),
-                          space.newtuple([subdtype, space.wrap(offset)]))
+                space.setitem(w_fields, space.newtext(name),
+                          space.newtuple([subdtype, space.newint(offset)]))
         return w_fields
 
     def descr_get_names(self, space):
         if not self.fields:
             return space.w_None
-        return space.newtuple([space.wrap(name[0]) for name in self.names])
+        return space.newtuple([space.newtext(name[0]) for name in self.names])
 
     def descr_set_names(self, space, w_names):
         if not self.fields:
@@ -462,30 +462,30 @@
         return False
 
     def descr_eq(self, space, w_other):
-        return space.wrap(self.eq(space, w_other))
+        return space.newbool(self.eq(space, w_other))
 
     def descr_ne(self, space, w_other):
-        return space.wrap(not self.eq(space, w_other))
+        return space.newbool(not self.eq(space, w_other))
 
     def descr_le(self, space, w_other):
         from .casting import can_cast_to
         w_other = as_dtype(space, w_other)
-        return space.wrap(can_cast_to(self, w_other))
+        return space.newbool(can_cast_to(self, w_other))
 
     def descr_ge(self, space, w_other):
         from .casting import can_cast_to
         w_other = as_dtype(space, w_other)
-        return space.wrap(can_cast_to(w_other, self))
+        return space.newbool(can_cast_to(w_other, self))
 
     def descr_lt(self, space, w_other):
         from .casting import can_cast_to
         w_other = as_dtype(space, w_other)
-        return space.wrap(can_cast_to(self, w_other) and not self.eq(space, w_other))
+        return space.newbool(can_cast_to(self, w_other) and not self.eq(space, w_other))
 
     def descr_gt(self, space, w_other):
         from .casting import can_cast_to
         w_other = as_dtype(space, w_other)
-        return space.wrap(can_cast_to(w_other, self) and not self.eq(space, w_other))
+        return space.newbool(can_cast_to(w_other, self) and not self.eq(space, w_other))
 
     def _compute_hash(self, space, x):
         from rpython.rlib.rarithmetic import intmask
@@ -518,7 +518,7 @@
         return x
 
     def descr_hash(self, space):
-        return space.wrap(self._compute_hash(space, 0x345678))
+        return space.newint(self._compute_hash(space, 0x345678))
 
     def descr_str(self, space):
         if self.fields:
@@ -540,12 +540,12 @@
 
     def descr_repr(self, space):
         if isinstance(self.itemtype, types.CharType):
-            return space.wrap("dtype('S1')")
+            return space.newtext("dtype('S1')")
         if self.fields:
             r = self.descr_get_descr(space, style='repr')
             name = space.str_w(space.str(self.w_box_type))
             if name != "<type 'numpy.void'>":
-                r = space.newtuple([space.wrap(self.w_box_type), r])
+                r = space.newtuple([self.w_box_type, r])
         elif self.subdtype is not None:
             r = space.newtuple([self.subdtype.descr_get_str(space),
                                 self.descr_get_shape(space)])
@@ -558,12 +558,12 @@
                 size = self.elsize
                 if self.num == NPY.UNICODE:
                     size >>= 2
-                r = space.wrap("'" + byteorder + self.char + str(size) + "'")
+                r = space.newtext("'" + byteorder + self.char + str(size) + "'")
             else:
                 r = self.descr_get_name(space, quote=True)
         if space.isinstance_w(r, space.w_str):
-            return space.wrap("dtype(%s)" % space.str_w(r))
-        return space.wrap("dtype(%s)" % space.str_w(space.repr(r)))
+            return space.newtext("dtype(%s)" % space.str_w(r))
+        return space.newtext("dtype(%s)" % space.str_w(space.repr(r)))
 
     def descr_getitem(self, space, w_item):
         if not self.fields:
@@ -589,8 +589,8 @@
 
     def descr_len(self, space):
         if not self.fields:
-            return space.wrap(0)
-        return space.wrap(len(self.fields))
+            return space.newint(0)
+        return space.newint(len(self.fields))
 
     def runpack_str(self, space, s):
         if self.is_str_or_unicode():
@@ -609,10 +609,10 @@
     def descr_reduce(self, space):
         w_class = space.type(self)
         builder_args = space.newtuple([
-            space.wrap("%s%d" % (self.kind, self.elsize)),
-            space.wrap(0), space.wrap(1)])
+            space.newbytes("%s%d" % (self.kind, self.elsize)),
+            space.newint(0), space.newint(1)])
 
-        version = space.wrap(3)
+        version = space.newint(3)
         endian = self.byteorder
         if endian == NPY.NATIVE:
             endian = NPY.NATBYTE
@@ -620,17 +620,17 @@
         names = self.descr_get_names(space)
         values = self.descr_get_fields(space)
         if self.is_flexible():
-            w_size = space.wrap(self.elsize)
+            w_size = space.newint(self.elsize)
             if self.alignment > 2:
-                w_alignment = space.wrap(self.alignment)
+                w_alignment = space.newint(self.alignment)
             else:
-                w_alignment = space.wrap(1)
+                w_alignment = space.newint(1)
         else:
-            w_size = space.wrap(-1)
-            w_alignment = space.wrap(-1)
-        w_flags = space.wrap(self.flags)
+            w_size = space.newint(-1)
+            w_alignment = space.newint(-1)
+        w_flags = space.newint(self.flags)
 
-        data = space.newtuple([version, space.wrap(endian), subdescr,
+        data = space.newtuple([version, space.newbytes(endian), subdescr,
                                names, values, w_size, w_alignment, w_flags])
         return space.newtuple([w_class, builder_args, data])
 
@@ -639,24 +639,24 @@
             # if builtin dtype (but not w_voiddtype)
             return space.w_None
 
-        version = space.int_w(space.getitem(w_data, space.wrap(0)))
+        version = space.int_w(space.getitem(w_data, space.newint(0)))
         if version != 3:
             raise oefmt(space.w_ValueError,
                         "can't handle version %d of numpy.dtype pickle",
                         version)
 
-        endian = byteorder_w(space, space.getitem(w_data, space.wrap(1)))
+        endian = byteorder_w(space, space.getitem(w_data, space.newint(1)))
         if endian == NPY.NATBYTE:
             endian = NPY.NATIVE
 
-        w_subarray = space.getitem(w_data, space.wrap(2))
-        w_names = space.getitem(w_data, space.wrap(3))
-        w_fields = space.getitem(w_data, space.wrap(4))
-        size = space.int_w(space.getitem(w_data, space.wrap(5)))
-        alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
+        w_subarray = space.getitem(w_data, space.newint(2))
+        w_names = space.getitem(w_data, space.newint(3))
+        w_fields = space.getitem(w_data, space.newint(4))
+        size = space.int_w(space.getitem(w_data, space.newint(5)))
+        alignment = space.int_w(space.getitem(w_data, space.newint(6)))
         if alignment < 2:
             alignment = -1
-        flags = space.int_w(space.getitem(w_data, space.wrap(7)))
+        flags = space.int_w(space.getitem(w_data, space.newint(7)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
             raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
@@ -685,11 +685,11 @@
             self.fields = {}
             for w_name in space.fixedview(w_names):
                 # XXX what happens if there is a title in the pickled dtype?
-                name = space.str_w(w_name)
+                name = space.bytes_w(w_name)
                 value = space.getitem(w_fields, w_name)
 
-                dtype = space.getitem(value, space.wrap(0))
-                offset = space.int_w(space.getitem(value, space.wrap(1)))
+                dtype = space.getitem(value, space.newint(0))
+                offset = space.int_w(space.getitem(value, space.newint(1)))
                 self.names.append((name, None))
                 assert isinstance(dtype, W_Dtype)
                 self.fields[name] = offset, dtype
@@ -829,7 +829,7 @@
     return retval
 
 def _get_val_or_none(space, w_dict, key):
-    w_key = space.wrap(key)
+    w_key = space.newtext(key)
     try:
         w_val = space.getitem(w_dict, w_key)
     except OperationError as e:
@@ -890,7 +890,7 @@
         try:
             return get_appbridge_cache(space).call_method(space,
                 'numpy.core._internal', '_usefields', Arguments(space, 
-                                [w_dict, space.wrap(alignment >= 0)]))
+                                [w_dict, space.newbool(alignment >= 0)]))
         except OperationError as e:
             if e.match(space, space.w_ImportError):
                 return _usefields(space, w_dict, alignment >= 0)
@@ -950,7 +950,7 @@
                     "_commastring is not returning a list with len >= 1")
     if space.len_w(w_lst) == 1:
         return descr__new__(space, space.gettypefor(W_Dtype),
-                            space.getitem(w_lst, space.wrap(0)), align=alignment>0)
+                            space.getitem(w_lst, space.newint(0)), align=alignment>0)
     else:
         try:
             return dtype_from_list(space, w_lst, True, alignment)
@@ -1088,15 +1088,15 @@
         return _set_metadata_and_copy( space, w_metadata,
                         dtype_from_list(space, w_dtype, False, alignment), copy)
     elif space.isinstance_w(w_dtype, space.w_tuple):
-        w_dtype0 = space.getitem(w_dtype, space.wrap(0))
-        w_dtype1 = space.getitem(w_dtype, space.wrap(1))
+        w_dtype0 = space.getitem(w_dtype, space.newint(0))
+        w_dtype1 = space.getitem(w_dtype, space.newint(1))
         # create a new dtype object
         l_side = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(l_side, W_Dtype)
         if l_side.elsize == 0 and space.isinstance_w(w_dtype1, space.w_int):
             #(flexible_dtype, itemsize)
             name = "%s%d" % (l_side.kind, space.int_w(w_dtype1))
-            retval = make_new_dtype(space, w_subtype, space.wrap(name), alignment, copy)
+            retval = make_new_dtype(space, w_subtype, space.newtext(name), alignment, copy)
             return _set_metadata_and_copy(space, w_metadata, retval, copy)
         elif (space.isinstance_w(w_dtype1, space.w_int) or
                 space.isinstance_w(w_dtype1, space.w_tuple) or 
@@ -1467,15 +1467,15 @@
                        space.wrap(dtype.itemtype.get_element_size())]
             if dtype.is_int():
                 if dtype.is_bool():
-                    w_maxobj = space.wrap(1)
-                    w_minobj = space.wrap(0)
+                    w_maxobj = space.newint(1)
+                    w_minobj = space.newint(0)
                 elif dtype.is_signed():
-                    w_maxobj = space.wrap(r_longlong((1 << (itembits - 1))
-                                          - 1))
-                    w_minobj = space.wrap(r_longlong(-1) << (itembits - 1))
+                    w_maxobj = space.newint(r_longlong((1 << (itembits - 1))
+                                            - 1))
+                    w_minobj = space.newint(r_longlong(-1) << (itembits - 1))
                 else:
-                    w_maxobj = space.wrap(r_ulonglong(1 << itembits) - 1)
-                    w_minobj = space.wrap(0)
+                    w_maxobj = space.newint(r_ulonglong(1 << itembits) - 1)
+                    w_minobj = space.newint(0)
                 items_w = items_w + [w_maxobj, w_minobj]
             items_w = items_w + [dtype.w_box_type]
             space.setitem(w_typeinfo, space.wrap(k), space.newtuple(items_w))
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -33,32 +33,32 @@
         return self
 
     def descr_c_contiguous(self, space):
-        return space.wrap(bool(self.flags & NPY.ARRAY_C_CONTIGUOUS))
+        return space.newbool(bool(self.flags & NPY.ARRAY_C_CONTIGUOUS))
 
     def descr_f_contiguous(self, space):
-        return space.wrap(bool(self.flags & NPY.ARRAY_F_CONTIGUOUS))
+        return space.newbool(bool(self.flags & NPY.ARRAY_F_CONTIGUOUS))
 
     def descr_get_writeable(self, space):
-        return space.wrap(bool(self.flags & NPY.ARRAY_WRITEABLE))
+        return space.newbool(bool(self.flags & NPY.ARRAY_WRITEABLE))
 
     def descr_get_owndata(self, space):
-        return space.wrap(bool(self.flags & NPY.ARRAY_OWNDATA))
+        return space.newbool(bool(self.flags & NPY.ARRAY_OWNDATA))
 
     def descr_get_aligned(self, space):
-        return space.wrap(bool(self.flags & NPY.ARRAY_ALIGNED))
+        return space.newbool(bool(self.flags & NPY.ARRAY_ALIGNED))
 
     def descr_get_fnc(self, space):
-        return space.wrap(bool(
+        return space.newbool(bool(
             self.flags & NPY.ARRAY_F_CONTIGUOUS and not
             self.flags & NPY.ARRAY_C_CONTIGUOUS ))
 
     def descr_get_forc(self, space):
-        return space.wrap(bool(
+        return space.newbool(bool(
             self.flags & NPY.ARRAY_F_CONTIGUOUS or
             self.flags & NPY.ARRAY_C_CONTIGUOUS ))
 
     def descr_get_num(self, space):
-        return space.wrap(self.flags)
+        return space.newint(self.flags)
 
     def descr_getitem(self, space, w_item):
         key = space.str_w(w_item)
@@ -83,10 +83,10 @@
         return self.flags == w_other.flags
 
     def descr_eq(self, space, w_other):
-        return space.wrap(self.eq(space, w_other))
+        return space.newbool(self.eq(space, w_other))
 
     def descr_ne(self, space, w_other):
-        return space.wrap(not self.eq(space, w_other))
+        return space.newbool(not self.eq(space, w_other))
 
     def descr___str__(self, space):
         s = StringBuilder()
@@ -102,7 +102,7 @@
         s.append(get_tf_str(self.flags, NPY.ARRAY_ALIGNED))
         s.append('\n  UPDATEIFCOPY : ')
         s.append(get_tf_str(self.flags, NPY.ARRAY_UPDATEIFCOPY))
-        return space.wrap(s.build())
+        return space.newtext(s.build())
 
 W_FlagsObject.typedef = TypeDef("numpy.flagsobj",
     __new__ = interp2app(W_FlagsObject.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -53,7 +53,7 @@
         return self
 
     def descr_len(self, space):
-        return space.wrap(self.iter.size)
+        return space.newint(self.iter.size)
 
     def descr_next(self, space):
         if self.iter.done(self.state):
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -71,7 +71,7 @@
 
     @jit.unroll_safe
     def get_index(self, space, shapelen):
-        return [space.wrap(self.indexes[i]) for i in range(shapelen)]
+        return [space.newint(self.indexes[i]) for i in range(shapelen)]
 
 
 class IterState(object):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -54,7 +54,7 @@
     @jit.unroll_safe
     def descr_get_shape(self, space):
         shape = self.get_shape()
-        return space.newtuple([space.wrap(i) for i in shape])
+        return space.newtuple([space.newint(i) for i in shape])
 
     def descr_set_shape(self, space, w_new_shape):
         shape = get_shape_from_iterable(space, self.get_size(), w_new_shape)
@@ -67,7 +67,7 @@
 
     def descr_get_strides(self, space):
         strides = self.implementation.get_strides()
-        return space.newtuple([space.wrap(i) for i in strides])
+        return space.newtuple([space.newint(i) for i in strides])
 
     def descr_get_dtype(self, space):
         return self.implementation.dtype
@@ -85,13 +85,13 @@
         raise oefmt(space.w_AttributeError, "Cannot delete array dtype")
 
     def descr_get_ndim(self, space):
-        return space.wrap(self.ndims())
+        return space.newint(self.ndims())
 
     def descr_get_itemsize(self, space):
-        return space.wrap(self.get_dtype().elsize)
+        return space.newint(self.get_dtype().elsize)
 
     def descr_get_nbytes(self, space):
-        return space.wrap(self.get_size() * self.get_dtype().elsize)
+        return space.newint(self.get_size() * self.get_dtype().elsize)
 
     def descr_fill(self, space, w_value):
         self.fill(space, self.get_dtype().coerce(space, w_value))
@@ -105,7 +105,7 @@
         arr = self
         if order != arr.get_order():
             arr = W_NDimArray(self.implementation.transpose(self, None))
-        return space.wrap(loop.tostring(space, arr))
+        return space.newtext(loop.tostring(space, arr))
 
     def getitem_filter(self, space, arr, axis=0):
         shape = self.get_shape()
@@ -122,7 +122,7 @@
                       " boolean dimension is %d" % (axis, shape[axis],
                       arr.get_shape()[0]))
                 #warning = space.gettypefor(support.W_VisibleDeprecationWarning)
-                space.warn(space.wrap(msg), space.w_VisibleDeprecationWarning)
+                space.warn(space.newtext(msg), space.w_VisibleDeprecationWarning)
             res_shape = shape[:axis] + [size] + shape[axis+1:]
         else:
             res_shape = [size]
@@ -207,7 +207,7 @@
                 copy = True
             w_ret = new_view(space, self, chunks)
             if copy:
-                w_ret = w_ret.descr_copy(space, space.wrap(w_ret.get_order()))
+                w_ret = w_ret.descr_copy(space, space.newint(w_ret.get_order()))
             return w_ret
         shape = res_shape + self.get_shape()[len(indexes):]
         w_res = W_NDimArray.from_shape(space, shape, self.get_dtype(),
@@ -231,7 +231,7 @@
                     dim = i
                     idx = c.w_idx
                     chunks.pop(i)
-                    chunks.insert(0, SliceChunk(space.newslice(space.wrap(0),
+                    chunks.insert(0, SliceChunk(space.newslice(space.newint(0),
                                  space.w_None, space.w_None)))
                     break
             if dim > 0:
@@ -268,7 +268,7 @@
             if isinstance(w_idx, boxes.W_IntegerBox):
                 # if w_idx is integer then getitem_array_int must contain a single value and we must return it.
                 # Get 0-th element of the w_ret.
-                w_ret = w_ret.implementation.descr_getitem(space, self, space.wrap(0))
+                w_ret = w_ret.implementation.descr_getitem(space, self, space.newint(0))
         else:
             try:
                 w_ret = self.implementation.descr_getitem(space, self, w_idx)
@@ -350,19 +350,19 @@
     def descr_len(self, space):
         shape = self.get_shape()
         if len(shape):
-            return space.wrap(shape[0])
+            return space.newint(shape[0])
         raise oefmt(space.w_TypeError, "len() of unsized object")
 
     def descr_repr(self, space):
         cache = get_appbridge_cache(space)
         if cache.w_array_repr is None:
-            return space.wrap(self.dump_data())
+            return space.newtext(self.dump_data())
         return space.call_function(cache.w_array_repr, self)
 
     def descr_str(self, space):
         cache = get_appbridge_cache(space)
         if cache.w_array_str is None:
-            return space.wrap(self.dump_data(prefix='', separator='', suffix=''))
+            return space.newtext(self.dump_data(prefix='', separator='', suffix=''))
         return space.call_function(cache.w_array_str, self)
 
     def dump_data(self, prefix='array(', separator=',', suffix=')'):
@@ -404,7 +404,7 @@
         self.implementation.fill(space, box)
 
     def descr_get_size(self, space):
-        return space.wrap(self.get_size())
+        return space.newint(self.get_size())
 
     def get_size(self):
         return self.implementation.get_size()
@@ -449,7 +449,7 @@
         if new_impl is not None:
             return wrap_impl(space, space.type(self), self, new_impl)
         # Create copy with contiguous data
-        arr = self.descr_copy(space, space.wrap(order))
+        arr = self.descr_copy(space, space.newint(order))
         if arr.get_size() > 0:
             new_implementation = arr.implementation.reshape(self, new_shape, order)
             if new_implementation is None:
@@ -562,7 +562,7 @@
             return self.get_scalar_value().item(space)
         l_w = []
         for i in range(self.get_shape()[0]):
-            item_w = self.descr_getitem(space, space.wrap(i))
+            item_w = self.descr_getitem(space, space.newint(i))
             if (isinstance(item_w, W_NDimArray) or
                     isinstance(item_w, boxes.W_GenericBox)):
                 l_w.append(space.call_method(item_w, "tolist"))
@@ -572,7 +572,7 @@
 
     def descr_ravel(self, space, w_order=None):
         order = order_converter(space, w_order, self.get_order())
-        return self.reshape(space, space.wrap(-1), order)
+        return self.reshape(space, space.newint(-1), order)
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None),
@@ -586,7 +586,7 @@
                         "axis unsupported for compress")
             arr = self
         else:
-            arr = self.reshape(space, space.wrap(-1), self.get_order())
+            arr = self.reshape(space, space.newint(-1), self.get_order())
         index = convert_to_array(space, w_obj)
         return arr.getitem_filter(space, index)
 
@@ -594,7 +594,7 @@
         order = order_converter(space, w_order, self.get_order())
         if self.is_scalar():
             # scalars have no storage
-            return self.reshape(space, space.wrap(1), order)
+            return self.reshape(space, space.newint(1), order)
         w_res = self.descr_ravel(space, w_order)
         if w_res.implementation.storage == self.implementation.storage:
             return w_res.descr_copy(space)
@@ -614,7 +614,7 @@
 
     def descr_get_flatiter(self, space):
         from .flatiter import W_FlatIterator
-        return space.wrap(W_FlatIterator(self))
+        return W_FlatIterator(self)
 
     def descr_item(self, space, args_w):
         if len(args_w) == 1 and space.isinstance_w(args_w[0], space.w_tuple):
@@ -674,7 +674,7 @@
             # will explode if it can't
             w_d = space.newdict()
             space.setitem_str(w_d, 'data',
-                              space.newtuple([space.wrap(addr), space.w_False]))
+                              space.newtuple([space.newint(addr), space.w_False]))
             space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
             space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
             if self.implementation.order == NPY.CORDER:
@@ -683,7 +683,7 @@
             else:
                 strides = self.descr_get_strides(space)
             space.setitem_str(w_d, 'strides', strides)
-            space.setitem_str(w_d, 'version', space.wrap(3))
+            space.setitem_str(w_d, 'version', space.newint(3))
             return w_d
 
     w_pypy_data = None
@@ -700,7 +700,7 @@
     __array_priority__ = 0.0
 
     def descr___array_priority__(self, space):
-        return space.wrap(self.__array_priority__)
+        return space.newfloat(self.__array_priority__)
 
     def descr_argsort(self, space, w_axis=None, w_kind=None, w_order=None):
         # happily ignore the kind
@@ -709,7 +709,7 @@
         # we would modify the array in-place. Use this to our advantage
         # by converting nonnative byte order.
         if self.is_scalar():
-            return space.wrap(0)
+            return space.newint(0)
         dtype = self.get_dtype().descr_newbyteorder(space, NPY.NATIVE)
         contig = self.implementation.astype(space, dtype, self.get_order())
         return contig.argsort(space, w_axis)
@@ -739,7 +739,7 @@
                         "according to the rule %s",
                         space.str_w(self.get_dtype().descr_repr(space)),
                         space.str_w(new_dtype.descr_repr(space)), casting)
-        order  = order_converter(space, space.wrap(order), self.get_order())
+        order  = order_converter(space, space.newtext(order), self.get_order())
         if (not copy and new_dtype == self.get_dtype()
                 and (order in (NPY.KEEPORDER, NPY.ANYORDER) or order == self.get_order())
                 and (subok or type(self) is W_NDimArray)):
@@ -838,7 +838,7 @@
     def descr_trace(self, space, offset=0, axis1=0, axis2=1,
                     w_dtype=None, w_out=None):
         diag = self.descr_diagonal(space, offset, axis1, axis2)
-        return diag.descr_sum(space, w_axis=space.wrap(-1), w_dtype=w_dtype, w_out=w_out)
+        return diag.descr_sum(space, w_axis=space.newint(-1), w_dtype=w_dtype, w_out=w_out)
 
     def descr_dump(self, space, w_file):
         raise oefmt(space.w_NotImplementedError, "dump not implemented yet")
@@ -1071,7 +1071,7 @@
                         "The truth value of an array with more than one "
                         "element is ambiguous. Use a.any() or a.all()")
         iter, state = self.create_iter()
-        return space.wrap(space.is_true(iter.getitem(state)))
+        return space.newbool(space.is_true(iter.getitem(state)))
 
     def _binop_impl(ufunc_name):
         def impl(self, space, w_other, w_out=None):
@@ -1174,11 +1174,11 @@
         elif self.ndims() < 2 and other.ndims() < 2:
             w_res = self.descr_mul(space, other)
             assert isinstance(w_res, W_NDimArray)
-            return w_res.descr_sum(space, space.wrap(-1), out)
+            return w_res.descr_sum(space, space.newint(-1), out)
         dtype = find_result_type(space, [self, other], [])
         if self.get_size() < 1 and other.get_size() < 1:
             # numpy compatability
-            return W_NDimArray.new_scalar(space, dtype, space.wrap(0))
+            return W_NDimArray.new_scalar(space, dtype, space.newint(0))
         # Do the dims match?
         out_shape, other_critical_dim = _match_dot_shapes(space, self, other)
         if out:
@@ -1243,8 +1243,8 @@
         def method(self, space, w_axis=None, w_dtype=None, w_out=None):
             out = out_converter(space, w_out)
             if space.is_none(w_axis):
-                w_axis = space.wrap(0)
-                arr = self.reshape(space, space.wrap(-1), self.get_order())
+                w_axis = space.newint(0)
+                arr = self.reshape(space, space.newint(-1), self.get_order())
             else:
                 arr = self
             ufunc = getattr(ufuncs.get(space), ufunc_name)
@@ -1359,7 +1359,7 @@
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
-            [space.wrap(0)]), space.wrap("b")])
+            [space.newint(0)]), space.wrap("b")])
 
         builder = StringBuilder()
         if self.get_dtype().is_object():
@@ -1377,11 +1377,11 @@
                                      self.implementation.get_storage_size())
 
         state = space.newtuple([
-            space.wrap(1),      # version
+            space.newint(1),      # version
             self.descr_get_shape(space),
             self.get_dtype(),
-            space.wrap(False),  # is_fortran
-            space.wrap(builder.build()),
+            space.newbool(False),  # is_fortran
+            space.newbytes(builder.build()),
         ])
 
         return space.newtuple([reconstruct, parameters, state])
@@ -1397,17 +1397,17 @@
             raise oefmt(space.w_ValueError,
                         "__setstate__ called with len(args[1])==%d, not 5 or 4",
                         lens)
-        shape = space.getitem(w_state, space.wrap(base_index))
-        dtype = space.getitem(w_state, space.wrap(base_index+1))
-        #isfortran = space.getitem(w_state, space.wrap(base_index+2))
-        storage = space.getitem(w_state, space.wrap(base_index+3))
+        shape = space.getitem(w_state, space.newint(base_index))
+        dtype = space.getitem(w_state, space.newint(base_index+1))
+        #isfortran = space.getitem(w_state, space.newint(base_index+2))
+        storage = space.getitem(w_state, space.newint(base_index+3))
         if not isinstance(dtype, descriptor.W_Dtype):
             raise oefmt(space.w_ValueError,
                         "__setstate__(self, (shape, dtype, .. called with "
                         "improper dtype '%R'", dtype)
         self.implementation = W_NDimArray.from_shape_and_storage(
             space, [space.int_w(i) for i in space.listview(shape)],
-            rffi.str2charp(space.str_w(storage), track_allocation=False),
+            rffi.str2charp(space.bytes_w(storage), track_allocation=False),
             dtype, storage_bytes=space.len_w(storage), owning=True).implementation
 
     def descr___array_finalize__(self, space, w_obj):
@@ -1470,7 +1470,7 @@
     w_ret = space.allocate_instance(W_NDimArray, w_subtype)
     W_NDimArray.__init__(w_ret, impl)
     space.call_function(space.getattr(w_ret,
-                        space.wrap('__array_finalize__')), w_subtype)
+                        space.newtext('__array_finalize__')), w_subtype)
     return w_ret
 
 
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -17,7 +17,7 @@
 
 def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
     if space.is_w(w_op_flags, space.w_None):
-        w_op_flags = space.newtuple([space.wrap('readonly')])
+        w_op_flags = space.newtuple([space.newtext('readonly')])
     if not space.isinstance_w(w_op_flags, space.w_tuple) and not \
             space.isinstance_w(w_op_flags, space.w_list):
         raise oefmt(space.w_ValueError,
@@ -510,7 +510,7 @@
             for i in range(len(self.seq)):
                 if i not in outargs:
                     self.seq[i] = self.seq[i].descr_copy(space,
-                                     w_order=space.wrap(self.order))
+                                     w_order=space.newint(self.order))
             self.dtypes = [s.get_dtype() for s in self.seq]
         else:
             #copy them from seq
@@ -592,7 +592,7 @@
         return oa_ndim
 
     def descr_iter(self, space):
-        return space.wrap(self)
+        return self
 
     def getitem(self, it, st):
         w_res = W_NDimArray(it.getoperand(st))
@@ -611,7 +611,7 @@
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_len(self, space):
-        space.wrap(len(self.iters))
+        space.newint(len(self.iters))
 
     @jit.unroll_safe
     def descr_next(self, space):
@@ -648,7 +648,7 @@
         return self.done
 
     def descr_iternext(self, space):
-        return space.wrap(self.iternext())
+        return space.newbool(self.iternext())
 
     def descr_copy(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
@@ -682,30 +682,30 @@
         return space.newtuple(res)
 
     def descr_get_finished(self, space):
-        return space.wrap(self.done)
+        return space.newbool(self.done)
 
     def descr_get_has_delayed_bufalloc(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_has_index(self, space):
-        return space.wrap(self.tracked_index in ["C", "F"])
+        return space.newbool(self.tracked_index in ["C", "F"])
 
     def descr_get_index(self, space):
         if not self.tracked_index in ["C", "F"]:
             raise oefmt(space.w_ValueError, "Iterator does not have an index")
         if self.done:
             raise oefmt(space.w_ValueError, "Iterator is past the end")
-        return space.wrap(self.index_iter.getvalue())
+        return space.newint(self.index_iter.getvalue())
 
     def descr_get_has_multi_index(self, space):
-        return space.wrap(self.tracked_index == "multi")
+        return space.newbool(self.tracked_index == "multi")
 
     def descr_get_multi_index(self, space):
         if not self.tracked_index == "multi":
             raise oefmt(space.w_ValueError, "Iterator is not tracking a multi-index")
         if self.done:
             raise oefmt(space.w_ValueError, "Iterator is past the end")
-        return space.newtuple([space.wrap(x) for x in self.index_iter.index])
+        return space.newtuple([space.newint(x) for x in self.index_iter.index])
 
     def descr_get_iterationneedsapi(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
@@ -714,13 +714,13 @@
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_itersize(self, space):
-        return space.wrap(support.product(self.shape))
+        return space.newint(support.product(self.shape))
 
     def descr_get_itviews(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
 
     def descr_get_ndim(self, space):
-        return space.wrap(self.ndim)
+        return space.newint(self.ndim)
 
     def descr_get_nop(self, space):
         raise oefmt(space.w_NotImplementedError, "not implemented yet")
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -98,11 +98,11 @@
     if dim >= 0:
         # filter by axis dim
         filtr = chunks[dim]
-        assert isinstance(filtr, BooleanChunk) 
+        assert isinstance(filtr, BooleanChunk)
         # XXX this creates a new array, and fails in setitem
         w_arr = w_arr.getitem_filter(space, filtr.w_idx, axis=dim)
         arr = w_arr.implementation
-        chunks[dim] = SliceChunk(space.newslice(space.wrap(0), 
+        chunks[dim] = SliceChunk(space.newslice(space.newint(0),
                                  space.w_None, space.w_None))
         r = calculate_slice_strides(space, arr.shape, arr.start,
                  arr.get_strides(), arr.get_backstrides(), chunks)
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -181,9 +181,9 @@
         return rffi.cast(lltype.Signed, storage) + start
 
 def is_rhs_priority_higher(space, w_lhs, w_rhs):
-    w_zero = space.wrap(0.0)
-    w_priority_l = space.findattr(w_lhs, space.wrap('__array_priority__')) or w_zero
-    w_priority_r = space.findattr(w_rhs, space.wrap('__array_priority__')) or w_zero
+    w_zero = space.newfloat(0.0)
+    w_priority_l = space.findattr(w_lhs, space.newtext('__array_priority__')) or w_zero
+    w_priority_r = space.findattr(w_rhs, space.newtext('__array_priority__')) or w_zero
     # XXX what is better, unwrapping values or space.gt?
     return space.is_true(space.gt(w_priority_r, w_priority_l))
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -196,7 +196,7 @@
         return w_obj
 
     def to_builtin_type(self, space, box):
-        return space.wrap(self.for_computation(self.unbox(box)))
+        raise NotImplementedError("has to be provided by subclass")
 
     def _coerce(self, space, w_item):
         raise NotImplementedError
@@ -398,7 +398,7 @@
         return self.box(space.is_true(w_item))
 
     def to_builtin_type(self, space, w_item):
-        return space.wrap(self.unbox(w_item))
+        return space.newbool(self.unbox(w_item))
 
     def str_format(self, box, add_quotes=True):
         return "True" if self.unbox(box) else "False"
@@ -461,6 +461,9 @@
     _mixin_ = True
     signed = True
 
+    def to_builtin_type(self, space, box):
+        return space.newint(self.for_computation(self.unbox(box)))
+
     def _base_coerce(self, space, w_item):
         if w_item is None:
             return self.box(0)
@@ -736,6 +739,9 @@
     _mixin_ = True
     strlen = 32
 
+    def to_builtin_type(self, space, box):
+        return space.wrap(self.for_computation(self.unbox(box)))
+
     def _coerce(self, space, w_item):
         if w_item is None:
             return self.box(0.0)
@@ -2015,17 +2021,17 @@
 
     @simple_binary_op
     def pow(self, v1, v2):
-        return self.space.pow(v1, v2, self.space.wrap(1))
+        return self.space.pow(v1, v2, self.space.newint(1))
 
     @simple_unary_op
     def reciprocal(self, v1):
-        return self.space.div(self.space.wrap(1.0), v1)
+        return self.space.div(self.space.newfloat(1.0), v1)
 
     @simple_unary_op
     def sign(self, v):
-        zero = self.space.wrap(0)
-        one = self.space.wrap(1)
-        m_one = self.space.wrap(-1)
+        zero = self.space.newint(0)
+        one = self.space.newint(1)
+        m_one = self.space.newint(-1)
         if self.space.is_true(self.space.gt(v, zero)):
             return one
         elif self.space.is_true(self.space.lt(v, zero)):
@@ -2158,7 +2164,7 @@
         if isinstance(w_item, boxes.W_StringBox):
             return w_item
         if w_item is None:
-            w_item = space.wrap('')
+            w_item = space.newbytes('')
         arg = space.str_w(space.str(w_item))
         arr = VoidBoxStorage(dtype.elsize, dtype)
         with arr as storage:
@@ -2196,7 +2202,7 @@
 
     # XXX move the rest of this to base class when UnicodeType is supported
     def to_builtin_type(self, space, box):
-        return space.wrap(self.to_str(box))
+        return space.newbytes(self.to_str(box))
 
     @str_binary_op
     def eq(self, v1, v2):
@@ -2317,7 +2323,7 @@
 
     def to_builtin_type(self, space, box):
         assert isinstance(box, boxes.W_UnicodeBox)
-        return space.wrap(box._value)
+        return space.newunicode(box._value)
 
     def eq(self, v1, v2):
         assert isinstance(v1, boxes.W_UnicodeBox)
@@ -2469,7 +2475,7 @@
                 read_val = dtype.read(item.arr, ofs, 0)
             if isinstance (read_val, boxes.W_StringBox):
                 # StringType returns a str
-                read_val = space.wrap(dtype.itemtype.to_str(read_val))
+                read_val = space.newbytes(dtype.itemtype.to_str(read_val))
             ret_unwrapped = ret_unwrapped + [read_val,]
         if len(ret_unwrapped) == 0:
             raise oefmt(space.w_NotImplementedError,
@@ -2490,7 +2496,7 @@
                 for i in range(len(dtype.names)):
                     name = dtype.names[i]
                     if name in w_item.dtype.names:
-                        items_w[i] = w_item.descr_getitem(space, space.wrap(name[0]))
+                        items_w[i] = w_item.descr_getitem(space, space.newtext(name[0]))
         elif w_item is not None:
             if space.isinstance_w(w_item, space.w_tuple):
                 if len(dtype.names) != space.len_w(w_item):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -105,10 +105,10 @@
         self.complex_to_float = complex_to_float
 
     def descr_get_name(self, space):
-        return space.wrap(self.name)
+        return space.newtext(self.name)
 
     def descr_repr(self, space):
-        return space.wrap("<ufunc '%s'>" % self.name)
+        return space.newtext("<ufunc '%s'>" % self.name)
 
     def get_doc(self, space):
         # Note: allows any object to be set as docstring, because why not?
@@ -160,7 +160,7 @@
 
     def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
         if w_axis is None:
-            w_axis = space.wrap(0)
+            w_axis = space.newint(0)
         out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, True, #keepdims must be true
                            out, w_dtype, variant=ACCUMULATE)
@@ -170,7 +170,7 @@
                      w_out=None, keepdims=False):
         from pypy.module.micronumpy.ndarray import W_NDimArray
         if w_axis is None:
-            w_axis = space.wrap(0)
+            w_axis = space.newint(0)
         out = out_converter(space, w_out)
         return self.reduce(space, w_obj, w_axis, keepdims, out, w_dtype)
 
@@ -207,7 +207,7 @@
                 axes[i] = x
         else:
             if space.isinstance_w(w_axis, space.w_tuple) and space.len_w(w_axis) == 1:
-                w_axis = space.getitem(w_axis, space.wrap(0))
+                w_axis = space.getitem(w_axis, space.newint(0))
             axis = space.int_w(w_axis)
             if axis < -shapelen or axis >= shapelen:
                 raise oefmt(space.w_ValueError, "'axis' entry is out of bounds")
@@ -373,7 +373,7 @@
             raise oefmt(space.w_ValueError,
                     "exactly two arguments expected")
         args = [convert_to_array(space, w_obj) for w_obj in args_w]
-        w_outshape = [space.wrap(i) for i in args[0].get_shape() + [1]*args[1].ndims()]
+        w_outshape = [space.newint(i) for i in args[0].get_shape() + [1]*args[1].ndims()]
         args0 = args[0].reshape(space, space.newtuple(w_outshape))
         return self.descr_call(space, Arguments.frompacked(space, 
                                                         space.newlist([args0, args[1]])))
@@ -391,7 +391,7 @@
         return w_subok, w_out, sig, w_casting, extobj_w
 
 def get_extobj(space):
-        extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
+        extobj_w = space.newlist([space.newint(8192), space.newint(0), space.w_None])
         return extobj_w
 
 
@@ -426,7 +426,7 @@
 def _has_reflected_op(space, w_obj, op):
     if op not in _reflected_ops:
         return False
-    return space.getattr(w_obj, space.wrap(_reflected_ops[op])) is not None
+    return space.getattr(w_obj, space.newtext(_reflected_ops[op])) is not None
 
 def safe_casting_mode(casting):
     assert casting is not None
@@ -475,7 +475,7 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            ctxt = space.newtuple([self, space.newtuple([w_obj]), space.wrap(0)])
+            ctxt = space.newtuple([self, space.newtuple([w_obj]), space.newint(0)])
             w_res = space.call_method(w_obj, '__array_wrap__', w_res, ctxt)
         return w_res
 
@@ -578,9 +578,9 @@
             elif ((w_ldtype.is_object() and w_ldtype.is_record()) or
                 (w_rdtype.is_object() and w_rdtype.is_record())):
                 if self.name == 'not_equal':
-                    return space.wrap(True)
+                    return space.w_True
                 elif self.name == 'equal':
-                    return space.wrap(False)
+                    return space.w_False
                 else:
                     msg = ("ufunc '%s' not supported for the input types, "
                            "and the inputs could not be safely coerced to "
@@ -595,19 +595,19 @@
         elif (w_ldtype.is_str()) and \
                 self.bool_result and out is None:
             if self.name in ('equal', 'less_equal', 'less'):
-               return space.wrap(False)
-            return space.wrap(True)
+               return space.w_False
+            return space.w_True
         elif (w_rdtype.is_str()) and \
                 self.bool_result and out is None:
             if self.name in ('not_equal','less', 'less_equal'):
-               return space.wrap(True)
-            return space.wrap(False)
+               return space.w_True
+            return space.w_False
         elif w_ldtype.is_flexible() or w_rdtype.is_flexible():
             if self.bool_result:
                 if self.name == 'equal' or self.name == 'not_equal':
                     res = w_ldtype.eq(space, w_rdtype)
                     if not res:
-                        return space.wrap(self.name == 'not_equal')
+                        return space.newbool(self.name == 'not_equal')
                 else:
                     return space.w_NotImplemented
             else:
@@ -641,7 +641,7 @@
         if out is None:
             if w_res.is_scalar():
                 return w_res.get_scalar_value()
-            ctxt = space.newtuple([self, space.newtuple([w_lhs, w_rhs]), space.wrap(0)])
+            ctxt = space.newtuple([self, space.newtuple([w_lhs, w_rhs]), space.newint(0)])
             w_res = space.call_method(w_highpriority, '__array_wrap__', w_res, ctxt)
         return w_res
 
@@ -907,7 +907,7 @@
             # from frompyfunc
             pass
         # mimic NpyIter_AdvancedNew with a nditer
-        w_itershape = space.newlist([space.wrap(i) for i in iter_shape])
+        w_itershape = space.newlist([space.newint(i) for i in iter_shape])
         nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                       w_op_flags, w_op_dtypes, w_casting, w_op_axes,
                       w_itershape, allow_backward=False)
@@ -956,7 +956,7 @@
                     outiters[0].descr_setitem(space, space.w_Ellipsis, outs)
                 else:
                     for i in range(self.nout):
-                        w_val = space.getitem(outs, space.wrap(i))
+                        w_val = space.getitem(outs, space.newint(i))
                         outiters[i].descr_setitem(space, space.w_Ellipsis, w_val)
         # XXX use _find_array_wrap and wrap outargs using __array_wrap__
         if len(outargs) > 1:
@@ -1531,7 +1531,7 @@
     if w_ret.external_loop:
         _parse_signature(space, w_ret, w_ret.signature)
     if doc:
-        w_ret.set_doc(space, space.wrap(doc))
+        w_ret.set_doc(space, space.newtext(doc))
     return w_ret
 
 # Instantiated in cpyext/ndarrayobject. It is here since ufunc calls
@@ -1572,8 +1572,8 @@
                 arg_i = args_w[i]
                 if not isinstance(arg_i, W_NDimArray):
                     raise OperationError(space.w_NotImplementedError,
-                         space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % (
-                                    arg_i, i)))
+                         space.newtext("cannot mix ndarray and %r (arg %d) in call to ufunc" % (
+                                       arg_i, i)))
                 with arg_i.implementation as storage:
                     addr = get_storage_as_int(storage, arg_i.get_start())
                     raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr))

From pypy.commits at gmail.com  Wed Nov  9 07:37:58 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:37:58 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: various fixes
Message-ID: <582318a6.88711c0a.1ea76.b8d1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88260:bfc7301d9344
Date: 2016-11-08 14:24 +0100
http://bitbucket.org/pypy/pypy/changeset/bfc7301d9344/

Log:	various fixes

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -478,7 +478,7 @@
     res, consumed, byteorder = runicode.str_decode_utf_16_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
     return space.newtuple([space.newunicode(res), space.newint(consumed),
-                           space.newtext(byteorder)])
+                           space.newint(byteorder)])
 
 @unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int,
              w_final=WrappedDefault(False))
@@ -497,7 +497,7 @@
     res, consumed, byteorder = runicode.str_decode_utf_32_helper(
         data, len(data), errors, final, state.decode_error_handler, byteorder)
     return space.newtuple([space.newunicode(res), space.newint(consumed),
-                           space.newtext(byteorder)])
+                           space.newint(byteorder)])
 
 # ____________________________________________________________
 # Charmap
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -22,7 +22,7 @@
     ]
 
 def _fetch(space, w_dialect, name):
-    return space.findattr(w_dialect, space.wrap(name))
+    return space.findattr(w_dialect, space.newtext(name))
 
 def _get_bool(space, w_src, default):
     if w_src is None:
@@ -134,7 +134,7 @@
                              w_escapechar, w_lineterminator, w_quotechar,
                              w_quoting, w_skipinitialspace, w_strict)
     if space.is_w(w_subtype, space.gettypeobject(W_Dialect.typedef)):
-        return space.wrap(dialect)
+        return dialect
     else:
         subdialect = space.allocate_instance(W_Dialect, w_subtype)
         subdialect.delimiter        = dialect.delimiter
@@ -145,18 +145,18 @@
         subdialect.quoting          = dialect.quoting
         subdialect.skipinitialspace = dialect.skipinitialspace
         subdialect.strict           = dialect.strict
-        return space.wrap(subdialect)
+        return subdialect
 
 
 def _get_escapechar(space, dialect):
     if dialect.escapechar == '\0':
         return space.w_None
-    return space.wrap(dialect.escapechar)
+    return space.newtext(dialect.escapechar)
 
 def _get_quotechar(space, dialect):
     if dialect.quotechar == '\0':
         return space.w_None
-    return space.wrap(dialect.quotechar)
+    return space.newtext(dialect.quotechar)
 
 
 W_Dialect.typedef = TypeDef(
diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py
--- a/pypy/module/_csv/interp_writer.py
+++ b/pypy/module/_csv/interp_writer.py
@@ -13,7 +13,7 @@
     def __init__(self, space, dialect, w_fileobj):
         self.space = space
         self.dialect = dialect
-        self.w_filewrite = space.getattr(w_fileobj, space.wrap('write'))
+        self.w_filewrite = space.getattr(w_fileobj, space.newtext('write'))
         # precompute this
         special = dialect.delimiter + dialect.lineterminator
         if dialect.escapechar != '\0':
@@ -26,8 +26,8 @@
     def error(self, msg):
         space = self.space
         w_module = space.getbuiltinmodule('_csv')
-        w_error = space.getattr(w_module, space.wrap('Error'))
-        raise OperationError(w_error, space.wrap(msg))
+        w_error = space.getattr(w_module, space.newtext('Error'))
+        raise OperationError(w_error, space.newtext(msg))
 
     def writerow(self, w_fields):
         """Construct and write a CSV record from a sequence of fields.
@@ -115,7 +115,7 @@
         rec.append(dialect.lineterminator)
 
         line = rec.build()
-        return space.call_function(self.w_filewrite, space.wrap(line))
+        return space.call_function(self.w_filewrite, space.newtext(line))
 
     def writerows(self, w_seqseq):
         """Construct and write a series of sequences to a csv file.
diff --git a/pypy/module/_demo/demo.py b/pypy/module/_demo/demo.py
--- a/pypy/module/_demo/demo.py
+++ b/pypy/module/_demo/demo.py
@@ -15,7 +15,7 @@
 
 def get(space, name):
     w_module = space.getbuiltinmodule('_demo')
-    return space.getattr(w_module, space.wrap(name))
+    return space.getattr(w_module, space.newtext(name))
 
 
 @unwrap_spec(repetitions=int)
@@ -27,7 +27,7 @@
     for i in range(repetitions):
         space.call_function(w_callable)
     endtime = time(0)
-    return space.wrap(endtime - starttime)
+    return space.newint(endtime - starttime)
 
 @unwrap_spec(n=int)
 def sieve(space, n):
@@ -55,10 +55,10 @@
     def multiply(self, w_y):
         space = self.space
         y = space.int_w(w_y)
-        return space.wrap(self.x * y)
+        return space.newint(self.x * y)
 
     def fget_x(self, space):
-        return space.wrap(self.x)
+        return space.newint(self.x)
 
     def fset_x(self, space, w_value):
         self.x = space.int_w(w_value)
@@ -66,8 +66,8 @@
 @unwrap_spec(x=int)
 def mytype_new(space, w_subtype, x):
     if x == 3:
-        return space.wrap(MySubType(space, x))
-    return space.wrap(W_MyType(space, x))
+        return MySubType(space, x)
+    return W_MyType(space, x)
 
 getset_x = GetSetProperty(W_MyType.fget_x, W_MyType.fset_x, cls=W_MyType)
 
diff --git a/pypy/module/_md5/__init__.py b/pypy/module/_md5/__init__.py
--- a/pypy/module/_md5/__init__.py
+++ b/pypy/module/_md5/__init__.py
@@ -21,7 +21,7 @@
     interpleveldefs = {
         'new': 'interp_md5.W_MD5',
         'MD5Type': 'interp_md5.W_MD5',
-        'digest_size': 'space.wrap(16)',
+        'digest_size': 'space.newint(16)',
         }
 
     appleveldefs = {
diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py
--- a/pypy/module/_md5/interp_md5.py
+++ b/pypy/module/_md5/interp_md5.py
@@ -28,7 +28,7 @@
     def copy_w(self):
         clone = W_MD5(self.space)
         clone._copyfrom(self)
-        return self.space.wrap(clone)
+        return clone
 
 
 @unwrap_spec(initialdata='bufferstr')
diff --git a/pypy/module/_multiprocessing/interp_memory.py b/pypy/module/_multiprocessing/interp_memory.py
--- a/pypy/module/_multiprocessing/interp_memory.py
+++ b/pypy/module/_multiprocessing/interp_memory.py
@@ -7,7 +7,7 @@
     if space.config.objspace.usemodules.mmap:
         mmap = space.interp_w(W_MMap, w_obj)
         address = rffi.cast(rffi.SIZE_T, mmap.mmap.data)
-        return space.newtuple([space.wrap(address),
-                               space.wrap(mmap.mmap.size)])
+        return space.newtuple([space.newint(address),
+                               space.newint(mmap.mmap.size)])
     else:
         raise oefmt(space.w_TypeError, "cannot get address of buffer")
diff --git a/pypy/module/_sha/__init__.py b/pypy/module/_sha/__init__.py
--- a/pypy/module/_sha/__init__.py
+++ b/pypy/module/_sha/__init__.py
@@ -21,9 +21,9 @@
     interpleveldefs = {
         'new': 'interp_sha.W_SHA',
         'SHAType': 'interp_sha.W_SHA',
-        'blocksize': 'space.wrap(1)',
-        'digest_size': 'space.wrap(20)',
-        'digestsize': 'space.wrap(20)',
+        'blocksize': 'space.newint(1)',
+        'digest_size': 'space.newint(20)',
+        'digestsize': 'space.newint(20)',
         }
 
     appleveldefs = {
diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py
--- a/pypy/module/_sre/__init__.py
+++ b/pypy/module/_sre/__init__.py
@@ -6,9 +6,9 @@
     }
 
     interpleveldefs = {
-        'CODESIZE':       'space.wrap(interp_sre.CODESIZE)',
+        'CODESIZE':       'space.newint(interp_sre.CODESIZE)',
         'MAGIC':          'space.newint(20031017)',
-        'MAXREPEAT':      'space.wrap(interp_sre.MAXREPEAT)',
+        'MAXREPEAT':      'space.newint(interp_sre.MAXREPEAT)',
         'compile':        'interp_sre.W_SRE_Pattern',
         'getlower':       'interp_sre.w_getlower',
         'getcodesize':    'interp_sre.w_getcodesize',
diff --git a/pypy/module/errno/interp_errno.py b/pypy/module/errno/interp_errno.py
--- a/pypy/module/errno/interp_errno.py
+++ b/pypy/module/errno/interp_errno.py
@@ -1,5 +1,5 @@
 import errno
 
 def get_errorcode(space):
-    return space.newint(errno.errorcode)
+    return space.wrap(errno.errorcode) # initializiation time
 
diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py
--- a/pypy/module/fcntl/interp_fcntl.py
+++ b/pypy/module/fcntl/interp_fcntl.py
@@ -111,7 +111,7 @@
             if rv < 0:
                 raise _get_error(space, "fcntl")
             arg = rffi.charpsize2str(ll_arg, len(arg))
-            return space.wrap(arg)
+            return space.newbytes(arg)
         finally:
             lltype.free(ll_arg, flavor='raw')
 
@@ -120,7 +120,7 @@
     rv = fcntl_int(fd, op, intarg)
     if rv < 0:
         raise _get_error(space, "fcntl")
-    return space.wrap(rv)
+    return space.newint(rv)
 
 @unwrap_spec(op=int)
 def flock(space, w_fd, op):
@@ -223,8 +223,8 @@
             arg = rffi.charpsize2str(ll_arg, len(arg))
             if mutate_flag != 0:
                 rwbuffer.setslice(0, arg)
-                return space.wrap(rv)
-            return space.wrap(arg)
+                return space.newint(rv)
+            return space.newbytes(arg)
         finally:
             lltype.free(ll_arg, flavor='raw')
 
@@ -245,7 +245,7 @@
             if rv < 0:
                 raise _get_error(space, "ioctl")
             arg = rffi.charpsize2str(ll_arg, len(arg))
-            return space.wrap(arg)
+            return space.newbytes(arg)
         finally:
             lltype.free(ll_arg, flavor='raw')
 
@@ -254,4 +254,4 @@
     rv = ioctl_int(fd, op, intarg)
     if rv < 0:
         raise _get_error(space, "ioctl")
-    return space.wrap(rv)
+    return space.newint(rv)
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -15,7 +15,7 @@
     cache = space.fromcache(MapAttrCache)
     cache.clear()
     rgc.collect()
-    return space.wrap(0)
+    return space.newint(0)
 
 def enable(space):
     """Non-recursive version.  Enable finalizers now.
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -128,7 +128,7 @@
             if WIN32 and signum not in signal_values:
                 self.handlers_w[signum] = space.w_None
             else:
-                self.handlers_w[signum] = space.wrap(SIG_DFL)
+                self.handlers_w[signum] = space.newint(SIG_DFL)
 
 def _get_handlers(space):
     return space.fromcache(Handlers).handlers_w
@@ -146,8 +146,8 @@
     pypysig_reinstall(n)
     # invoke the app-level handler
     ec = space.getexecutioncontext()
-    w_frame = space.wrap(ec.gettopframe_nohidden())
-    space.call_function(w_handler, space.wrap(n), w_frame)
+    w_frame = ec.gettopframe_nohidden()
+    space.call_function(w_handler, space.newint(n), w_frame)
 
 
 @unwrap_spec(signum=int)
@@ -179,7 +179,7 @@
 @jit.dont_look_inside
 @unwrap_spec(timeout=int)
 def alarm(space, timeout):
-    return space.wrap(c_alarm(timeout))
+    return space.newint(c_alarm(timeout))
 
 
 @jit.dont_look_inside
@@ -216,9 +216,9 @@
                     "__pypy__.thread.enable_signals()")
     check_signum_in_range(space, signum)
 
-    if space.eq_w(w_handler, space.wrap(SIG_DFL)):
+    if space.eq_w(w_handler, space.newint(SIG_DFL)):
         pypysig_default(signum)
-    elif space.eq_w(w_handler, space.wrap(SIG_IGN)):
+    elif space.eq_w(w_handler, space.newint(SIG_IGN)):
         pypysig_ignore(signum)
     else:
         if not space.is_true(space.callable(w_handler)):
@@ -252,7 +252,7 @@
             if e.errno == errno.EBADF:
                 raise oefmt(space.w_ValueError, "invalid fd")
     old_fd = pypysig_set_wakeup_fd(fd, True)
-    return space.wrap(intmask(old_fd))
+    return space.newint(intmask(old_fd))
 
 
 @jit.dont_look_inside
@@ -261,7 +261,7 @@
     check_signum_in_range(space, signum)
     if rffi.cast(lltype.Signed, c_siginterrupt(signum, flag)) < 0:
         errno = rposix.get_saved_errno()
-        raise OperationError(space.w_RuntimeError, space.wrap(errno))
+        raise OperationError(space.w_RuntimeError, space.newint(errno))
 
 
 #__________________________________________________________
@@ -277,8 +277,8 @@
 
 
 def itimer_retval(space, val):
-    w_value = space.wrap(double_from_timeval(val.c_it_value))
-    w_interval = space.wrap(double_from_timeval(val.c_it_interval))
+    w_value = space.newfloat(double_from_timeval(val.c_it_value))
+    w_interval = space.newfloat(double_from_timeval(val.c_it_interval))
     return space.newtuple([w_value, w_interval])
 
 
diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py
--- a/pypy/module/thread/os_thread.py
+++ b/pypy/module/thread/os_thread.py
@@ -189,7 +189,7 @@
             raise
     except rthread.error:
         raise wrap_thread_error(space, "can't start new thread")
-    return space.wrap(ident)
+    return space.newint(ident)
 
 
 def get_ident(space):
@@ -201,7 +201,7 @@
 be relied upon, and the number should be seen purely as a magic cookie.
 A thread's identity may be reused for another thread after it exits."""
     ident = rthread.get_ident()
-    return space.wrap(ident)
+    return space.newint(ident)
 
 @unwrap_spec(size=int)
 def stack_size(space, size=0):
@@ -231,7 +231,7 @@
         raise oefmt(space.w_ValueError, "size not valid: %d bytes", size)
     if error == -2:
         raise wrap_thread_error(space, "setting stack size not supported")
-    return space.wrap(old_size)
+    return space.newint(old_size)
 
 def _count(space):
     """_count() -> integer
@@ -242,7 +242,7 @@
 
 This function is meant for internal and specialized purposes only.
 In most applications `threading.enumerate()` should be used instead."""
-    return space.wrap(bootstrapper.nbthreads)
+    return space.newint(bootstrapper.nbthreads)
 
 def exit(space):
     """This is synonymous to ``raise SystemExit''.  It will cause the current

From pypy.commits at gmail.com  Wed Nov  9 07:38:00 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:38:00 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _cffi_backend (there's a bug
 somewhere)
Message-ID: <582318a8.8a29c20a.4d71c.caf9@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88261:ff029105d0dc
Date: 2016-11-08 14:26 +0100
http://bitbucket.org/pypy/pypy/changeset/ff029105d0dc/

Log:	_cffi_backend (there's a bug somewhere)

diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -27,7 +27,7 @@
             return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
         else:
             w_raw_cdata = space.call_function(self.w_alloc,
-                                              space.wrap(datasize))
+                                              space.newint(datasize))
             if not isinstance(w_raw_cdata, cdataobj.W_CData):
                 raise oefmt(space.w_TypeError,
                             "alloc() must return a cdata object (got %T)",
@@ -76,7 +76,7 @@
     if w_alloc is None and w_free is not None:
         raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
     alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
-    return space.wrap(alloc)
+    return alloc
 
 
 default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py
--- a/pypy/module/_cffi_backend/call_python.py
+++ b/pypy/module/_cffi_backend/call_python.py
@@ -90,7 +90,7 @@
     ffi = space.interp_w(W_FFIObject, w_ffi)
 
     if space.is_w(w_name, space.w_None):
-        w_name = space.getattr(w_python_callable, space.wrap('__name__'))
+        w_name = space.getattr(w_python_callable, space.newtext('__name__'))
     name = space.str_w(w_name)
 
     ctx = ffi.ctxobj.ctx
@@ -130,4 +130,4 @@
 
 @specialize.memo()
 def get_generic_decorator(space):
-    return space.wrap(interp2app(externpy_deco))
+    return space.wrap(interp2app(externpy_deco)) # init time
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -66,13 +66,13 @@
             from pypy.module._cffi_backend import ctypestruct
             if isinstance(self.ctype, ctypestruct.W_CTypeStructOrUnion):
                 extra1 = ' &'
-        return self.space.wrap("<cdata '%s%s' %s>" % (
+        return self.space.newtext("<cdata '%s%s' %s>" % (
             self.ctype.name, extra1, extra2))
 
     def nonzero(self):
         with self as ptr:
             nonzero = self.ctype.nonzero(ptr)
-        return self.space.wrap(nonzero)
+        return self.space.newbool(nonzero)
 
     def int(self, space):
         with self as ptr:
@@ -95,7 +95,7 @@
         from pypy.module._cffi_backend import ctypearray
         space = self.space
         if isinstance(self.ctype, ctypearray.W_CTypeArray):
-            return space.wrap(self.get_array_length())
+            return space.newint(self.get_array_length())
         raise oefmt(space.w_TypeError,
                     "cdata of type '%s' has no len()", self.ctype.name)
 
@@ -137,7 +137,7 @@
         # alignment (to 4, 8, maybe 16 bytes), so we use the following
         # formula to avoid the trailing bits being always 0.
         h = h ^ (h >> 4)
-        return self.space.wrap(h)
+        return self.space.newint(h)
 
     def getitem(self, w_index):
         space = self.space
@@ -319,7 +319,7 @@
                         "pointer subtraction: the distance between the two "
                         "pointers is not a multiple of the item size")
                 diff //= itemsize
-            return space.wrap(diff)
+            return space.newint(diff)
         #
         return self._add_or_sub(w_other, -1)
 
@@ -430,7 +430,7 @@
         if isinstance(ct, W_CTypePointer):
             ct = ct.ctitem
         lst = ct.cdata_dir()
-        return space.newlist([space.wrap(s) for s in lst])
+        return space.newlist([space.newtext(s) for s in lst])
 
     def get_structobj(self):
         return None
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,7 +13,7 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_alterrno())
+    return space.newint(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
@@ -27,4 +27,4 @@
     if code == -1:
         code = GetLastError_alt_saved()
     message = FormatError(code)
-    return space.newtuple([space.wrap(code), space.wrap(message)])
+    return space.newtuple([space.newint(code), space.newtext(message)])
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -39,12 +39,12 @@
     if src_ctx.c_includes:
         lib.make_includes_from(src_ctx.c_includes)
 
-    w_name = space.wrap(name)
+    w_name = space.newtext(name)
     module = Module(space, w_name)
     if path is not None:
-        module.setdictvalue(space, '__file__', space.wrap(path))
-    module.setdictvalue(space, 'ffi', space.wrap(ffi))
-    module.setdictvalue(space, 'lib', space.wrap(lib))
+        module.setdictvalue(space, '__file__', space.newtext(path))
+    module.setdictvalue(space, 'ffi', space.newtext(ffi))
+    module.setdictvalue(space, 'lib', space.newtext(lib))
     w_modules_dict = space.sys.get('modules')
-    space.setitem(w_modules_dict, w_name, space.wrap(module))
-    space.setitem(w_modules_dict, space.wrap(name + '.lib'), space.wrap(lib))
+    space.setitem(w_modules_dict, w_name, space.newtext(module))
+    space.setitem(w_modules_dict, space.newtext(name + '.lib'), space.newtext(lib))
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -100,7 +100,7 @@
             return self.space.wrap(self.ctitem)
         if attrchar == 'l':     # length
             if self.length >= 0:
-                return self.space.wrap(self.length)
+                return self.space.newint(self.length)
             else:
                 return self.space.w_None
         return W_CTypePtrOrArray._fget(self, attrchar)
@@ -121,7 +121,7 @@
         self._stop = rffi.ptradd(self._next, length * ctitem.size)
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         result = self._next
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -120,11 +120,11 @@
             return self.space.newtuple([self.space.wrap(a)
                                         for a in self.fargs])
         if attrchar == 'r':    # result
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         if attrchar == 'E':    # ellipsis
             return self.space.wrap(self.ellipsis)
         if attrchar == 'A':    # abi
-            return self.space.wrap(self.abi)
+            return self.space.newint(self.abi)
         return W_CTypePtrBase._fget(self, attrchar)
 
     def call(self, funcaddr, args_w):
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -34,7 +34,7 @@
 
     def repr(self):
         space = self.space
-        return space.wrap("<ctype '%s'>" % (self.name,))
+        return space.newtext("<ctype '%s'>" % (self.name,))
 
     def extra_repr(self, cdata):
         if cdata:
@@ -229,17 +229,17 @@
     # __________ app-level attributes __________
     def dir(self):
         space = self.space
-        lst = [space.wrap(name)
+        lst = [space.newtext(name)
                   for name in _name_of_attributes
-                  if space.findattr(self, space.wrap(name)) is not None]
+                  if space.findattr(self, space.newtext(name)) is not None]
         return space.newlist(lst)
 
     def _fget(self, attrchar):
         space = self.space
         if attrchar == 'k':     # kind
-            return space.wrap(self.kind)      # class attribute
+            return space.newtext(self.kind)      # class attribute
         if attrchar == 'c':     # cname
-            return space.wrap(self.name)
+            return space.newtext(self.name)
         raise oefmt(space.w_AttributeError,
                     "ctype '%s' has no such attribute", self.name)
 
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -122,7 +122,7 @@
     _attrs_ = []
 
     def cast_to_int(self, cdata):
-        return self.space.wrap(ord(cdata[0]))
+        return self.space.newint(ord(cdata[0]))
 
     def convert_to_object(self, cdata):
         return self.space.newbytes(cdata[0])
@@ -166,11 +166,11 @@
 
     def cast_to_int(self, cdata):
         unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.wrap(unichardata[0])
+        return self.space.newint(unichardata[0])
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.wrap(unichardata[0])
+        return self.space.newunicode(unichardata[0])
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
@@ -195,7 +195,7 @@
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
-        return self.space.wrap(u)
+        return self.space.newunicode(u)
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
@@ -214,14 +214,14 @@
     def convert_to_object(self, cdata):
         if self.value_fits_long:
             value = misc.read_raw_long_data(cdata, self.size)
-            return self.space.wrap(value)
+            return self.space.newint(value)
         else:
             return self._convert_to_object_longlong(cdata)
 
     def _convert_to_object_longlong(self, cdata):
         # in its own function: LONGLONG may make the whole function jit-opaque
         value = misc.read_raw_signed_data(cdata, self.size)
-        return self.space.wrap(value)    # r_longlong => on 32-bit, 'long'
+        return self.space.newint(value)    # r_longlong => on 32-bit, 'long'
 
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
@@ -271,7 +271,7 @@
                 overflowed = misc.pack_list_to_raw_array_bounds_signed(
                     int_list, cdata, self.size)
                 if overflowed != 0:
-                    self._overflow(self.space.wrap(overflowed))
+                    self._overflow(self.space.newint(overflowed))
             return True
         return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
@@ -316,16 +316,16 @@
         if self.value_fits_ulong:
             value = misc.read_raw_ulong_data(cdata, self.size)
             if self.value_fits_long:
-                return self.space.wrap(intmask(value))
+                return self.space.newint(intmask(value))
             else:
-                return self.space.wrap(value)    # r_uint => 'long' object
+                return self.space.newint(value)    # r_uint => 'long' object
         else:
             return self._convert_to_object_longlong(cdata)
 
     def _convert_to_object_longlong(self, cdata):
         # in its own function: LONGLONG may make the whole function jit-opaque
         value = misc.read_raw_unsigned_data(cdata, self.size)
-        return self.space.wrap(value)    # r_ulonglong => 'long' object
+        return self.space.newint(value)    # r_ulonglong => 'long' object
 
     def get_vararg_type(self):
         if self.size < rffi.sizeof(rffi.INT):
@@ -349,7 +349,7 @@
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
-                self._overflow(self.space.wrap(overflowed))
+                self._overflow(self.space.newint(overflowed))
             return True
         return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
 
@@ -406,7 +406,7 @@
 
     def convert_to_object(self, cdata):
         value = misc.read_raw_float_data(cdata, self.size)
-        return self.space.wrap(value)
+        return self.space.newfloat(value)
 
     def convert_from_object(self, cdata, w_ob):
         space = self.space
@@ -486,7 +486,7 @@
 
     def float(self, cdata):
         value = self._read_from_longdouble(cdata)
-        return self.space.wrap(value)
+        return self.space.newfloat(value)
 
     def convert_to_object(self, cdata):
         w_cdata = cdataobj.W_CDataMem(self.space, self)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -165,7 +165,7 @@
                     "will be forbidden in the future (check that the types "
                     "are as you expect; use an explicit ffi.cast() if they "
                     "are correct)" % (other.name, self.name))
-                space.warn(space.wrap(msg), space.w_UserWarning)
+                space.warn(space.newtext(msg), space.w_UserWarning)
             else:
                 raise self._convert_error("compatible pointer", w_ob)
 
@@ -368,7 +368,7 @@
 
     def _fget(self, attrchar):
         if attrchar == 'i':     # item
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         return W_CTypePtrBase._fget(self, attrchar)
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -69,8 +69,8 @@
             result = [None] * len(self._fields_list)
             for fname, field in self._fields_dict.iteritems():
                 i = self._fields_list.index(field)
-                result[i] = space.newtuple([space.wrap(fname),
-                                            space.wrap(field)])
+                result[i] = space.newtuple([space.newtext(fname),
+                                            field])
             return space.newlist(result)
         return W_CType._fget(self, attrchar)
 
@@ -92,7 +92,7 @@
         try:
             cfield = self._getcfield_const(fieldname)
         except KeyError:
-            raise OperationError(space.w_KeyError, space.wrap(fieldname))
+            raise OperationError(space.w_KeyError, space.newtext(fieldname))
         if cfield.bitshift >= 0:
             raise oefmt(space.w_TypeError, "not supported for bitfields")
         return (cfield.ctype, cfield.offset)
@@ -279,14 +279,14 @@
                 shiftforsign = r_uint(1) << (self.bitsize - 1)
                 value = ((value >> self.bitshift) + shiftforsign) & valuemask
                 result = intmask(value) - intmask(shiftforsign)
-                return space.wrap(result)
+                return space.newint(result)
             else:
                 value = misc.read_raw_unsigned_data(cdata, ctype.size)
                 valuemask = (r_ulonglong(1) << self.bitsize) - 1
                 shiftforsign = r_ulonglong(1) << (self.bitsize - 1)
                 value = ((value >> self.bitshift) + shiftforsign) & valuemask
                 result = r_longlong(value) - r_longlong(shiftforsign)
-                return space.wrap(result)
+                return space.newint(result)
         #
         if isinstance(ctype, ctypeprim.W_CTypePrimitiveUnsigned):
             value_fits_long = ctype.value_fits_long
@@ -302,14 +302,14 @@
             valuemask = (r_uint(1) << self.bitsize) - 1
             value = (value >> self.bitshift) & valuemask
             if value_fits_long:
-                return space.wrap(intmask(value))
+                return space.newint(intmask(value))
             else:
-                return space.wrap(value)    # uint => wrapped long object
+                return space.newint(value)    # uint => wrapped long object
         else:
             value = misc.read_raw_unsigned_data(cdata, ctype.size)
             valuemask = (r_ulonglong(1) << self.bitsize) - 1
             value = (value >> self.bitshift) & valuemask
-            return space.wrap(value)      # ulonglong => wrapped long object
+            return space.newint(value)      # ulonglong => wrapped long object
 
     def convert_bitfield_from_object(self, cdata, w_ob):
         ctype = self.ctype
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -37,7 +37,7 @@
     compiler = space.createcompiler()
     pycode = compiler.compile(code, "<init code for '%s'>" % name, 'exec', 0)
     w_globals = space.newdict(module=True)
-    space.setitem_str(w_globals, "__builtins__", space.wrap(space.builtin))
+    space.setitem_str(w_globals, "__builtins__", space.builtin)
     pycode.exec_code(space, w_globals, w_globals)
 
 
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -46,17 +46,17 @@
                         "ctype '%s' is of unknown size", w_obj.name)
     else:
         raise oefmt(space.w_TypeError, "expected a 'cdata' or 'ctype' object")
-    return space.wrap(size)
+    return space.newint(size)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def alignof(space, w_ctype):
     align = w_ctype.alignof()
-    return space.wrap(align)
+    return space.newint(align)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, following=int)
 def typeoffsetof(space, w_ctype, w_field_or_index, following=0):
     ctype, offset = w_ctype.direct_typeoffsetof(w_field_or_index, following)
-    return space.newtuple([space.wrap(ctype), space.wrap(offset)])
+    return space.newtuple([space.wrap(ctype), space.newint(offset)])
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_cdata=cdataobj.W_CData, offset=int)
 def rawaddressof(space, w_ctype, w_cdata, offset):
@@ -68,7 +68,7 @@
 def getcname(space, w_ctype, replace_with):
     p = w_ctype.name_position
     s = '%s%s%s' % (w_ctype.name[:p], replace_with, w_ctype.name[p:])
-    return space.wrap(s)
+    return space.newtext(s)
 
 # ____________________________________________________________
 
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -26,7 +26,7 @@
         self.libname = libname    # some string that gives the name of the lib
 
     def descr_repr(self):
-        return self.space.wrap("<Lib object for '%s'>" % self.libname)
+        return self.space.newtext("<Lib object for '%s'>" % self.libname)
 
     def make_includes_from(self, c_includes):
         space = self.space
@@ -35,7 +35,7 @@
         while c_includes[num]:
             include_name = rffi.charp2str(c_includes[num])
             try:
-                w_lib1 = space.appexec([space.wrap(include_name)], """(modname):
+                w_lib1 = space.appexec([space.newtext(include_name)], """(modname):
                     mod = __import__(modname, None, None, ['ffi', 'lib'])
                     return mod.lib""")
                 lib1 = space.interp_w(W_LibObject, w_lib1)
@@ -185,7 +185,7 @@
         return w_result
 
     def _get_attr(self, w_attr, is_getattr=False):
-        attr = self.space.str_w(w_attr)
+        attr = self.space.text_w(w_attr)
         try:
             w_value = self._get_attr_elidable(attr)
         except KeyError:
@@ -202,7 +202,7 @@
                     from pypy.interpreter.module import Module
                     return self.space.gettypeobject(Module.typedef)
                 if is_getattr and attr == '__name__':
-                    return self.space.wrap("%s.lib" % self.libname)
+                    return self.space.newtext("%s.lib" % self.libname)
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",
diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -36,7 +36,7 @@
 
     def repr(self):
         space = self.space
-        return space.wrap("<clibrary '%s'>" % self.name)
+        return space.newtext("<clibrary '%s'>" % self.name)
 
     @unwrap_spec(w_ctype=W_CType, name=str)
     def load_function(self, w_ctype, name):
@@ -97,4 +97,4 @@
 @unwrap_spec(filename="str_or_None", flags=int)
 def load_library(space, filename, flags=0):
     lib = W_Library(space, filename, flags)
-    return space.wrap(lib)
+    return lib
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -141,7 +141,7 @@
     try:
         return bigint.tolonglong()
     except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+        raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
 
 def as_long(space, w_ob):
     # Same as as_long_long(), but returning an int instead.
@@ -158,7 +158,7 @@
     try:
         return bigint.toint()
     except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+        raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
 
 def as_unsigned_long_long(space, w_ob, strict):
     # (possibly) convert and cast a Python object to an unsigned long long.
@@ -168,7 +168,7 @@
     if space.is_w(space.type(w_ob), space.w_int):   # shortcut
         value = space.int_w(w_ob)
         if strict and value < 0:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         return r_ulonglong(value)
     try:
         bigint = space.bigint_w(w_ob, allow_conversion=False)
@@ -182,9 +182,9 @@
         try:
             return bigint.toulonglong()
         except ValueError:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         except OverflowError:
-            raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
     else:
         return bigint.ulonglongmask()
 
@@ -193,7 +193,7 @@
     if space.is_w(space.type(w_ob), space.w_int):   # shortcut
         value = space.int_w(w_ob)
         if strict and value < 0:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         return r_uint(value)
     try:
         bigint = space.bigint_w(w_ob, allow_conversion=False)
@@ -207,9 +207,9 @@
         try:
             return bigint.touint()
         except ValueError:
-            raise OperationError(space.w_OverflowError, space.wrap(neg_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(neg_msg))
         except OverflowError:
-            raise OperationError(space.w_OverflowError, space.wrap(ovf_msg))
+            raise OperationError(space.w_OverflowError, space.newtext(ovf_msg))
     else:
         return bigint.uintmask()
 
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -146,7 +146,7 @@
     try:
         ctypecls, size, align = PRIMITIVE_TYPES[name]
     except KeyError:
-        raise OperationError(space.w_KeyError, space.wrap(name))
+        raise OperationError(space.w_KeyError, space.newtext(name))
     ctype = ctypecls(space, size, name, len(name), align)
     unique_cache.primitives[name] = ctype
     return ctype
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -131,15 +131,15 @@
 
     if neg == 0:     # positive
         if value <= rffi.cast(rffi.ULONGLONG, sys.maxint):
-            return ffi.space.wrap(intmask(value))
+            return ffi.space.newint(intmask(value))
         else:
-            return ffi.space.wrap(value)
+            return ffi.space.newint(value)
     elif neg == 1:   # negative
         value = rffi.cast(rffi.LONGLONG, value)
         if value >= -sys.maxint-1:
-            return ffi.space.wrap(intmask(value))
+            return ffi.space.newint(intmask(value))
         else:
-            return ffi.space.wrap(value)
+            return ffi.space.newint(value)
 
     if neg == 2:
         got = "%d (0x%x)" % (value, value)
@@ -364,7 +364,7 @@
             while p[j] != ',' and p[j] != '\x00':
                 j += 1
             enname = rffi.charpsize2str(p, j)
-            enumerators_w.append(space.wrap(enname))
+            enumerators_w.append(space.newtext(enname))
 
             gindex = parse_c_type.search_in_globals(ffi.ctxobj.ctx, enname)
             assert gindex >= 0
@@ -493,10 +493,10 @@
                                          field_name, "'")
 
         fields_w[i] = space.newtuple([
-            space.wrap(field_name),
+            space.newtext(field_name),
             w_ctf,
-            space.wrap(fbitsize),
-            space.wrap(field_offset)])
+            space.newint(fbitsize),
+            space.newint(field_offset)])
 
     sflags = 0
     c_flags = rffi.getintfield(s, 'c_flags')
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1203,20 +1203,6 @@
     orig_getline = linecache.getline
     try:
         linecache.getline = lambda *args: 'LINE'    # hack: speed up PyPy tests
-        sys.stderr = cStringIO.StringIO()
-        assert f(100) == 300
-        assert sys.stderr.getvalue() == ''
-        assert f(10000) == -42
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Traceback (most recent call last):
-  File "$", line $, in Zcb1
-    $
-  File "$", line $, in check_value
-    $
-ValueError: 42
-""")
-        sys.stderr = cStringIO.StringIO()
         bigvalue = 20000
         assert f(bigvalue) == -42
         assert matches(sys.stderr.getvalue(), """\
@@ -1224,59 +1210,6 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        assert len(seen) == 0
-        assert ff(bigvalue) == -42
-        assert sys.stderr.getvalue() == ""
-        assert len(seen) == 1
-        exc, val, tb = seen[0]
-        assert exc is OverflowError
-        assert str(val) == "integer 60000 does not fit 'short'"
-        #
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        del seen[:]
-        oops_result = 81
-        assert ff(bigvalue) == 81
-        oops_result = None
-        assert sys.stderr.getvalue() == ""
-        assert len(seen) == 1
-        exc, val, tb = seen[0]
-        assert exc is OverflowError
-        assert str(val) == "integer 60000 does not fit 'short'"
-        #
-        sys.stderr = cStringIO.StringIO()
-        bigvalue = 20000
-        del seen[:]
-        oops_result = "xy"     # not None and not an int!
-        assert ff(bigvalue) == -42
-        oops_result = None
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Trying to convert the result back to C:
-OverflowError: integer 60000 does not fit 'short'
-
-During the call to 'onerror', another exception occurred:
-
-TypeError: $integer$
-""")
-        #
-        sys.stderr = cStringIO.StringIO()
-        seen = "not a list"    # this makes the oops() function crash
-        assert ff(bigvalue) == -42
-        assert matches(sys.stderr.getvalue(), """\
-From cffi callback <function$Zcb1 at 0x$>:
-Trying to convert the result back to C:
-OverflowError: integer 60000 does not fit 'short'
-
-During the call to 'onerror', another exception occurred:
-
-Traceback (most recent call last):
-  File "$", line $, in oops
-    $
-AttributeError: 'str' object has no attribute 'append'
-""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -93,12 +93,12 @@
 
     def descr_repr(self, space):
         doc = self.rawfunctype.repr_fn_type(self.ffi, self.fnname)
-        return space.wrap("<FFIFunctionWrapper '%s'>" % (doc,))
+        return space.newtext("<FFIFunctionWrapper '%s'>" % (doc,))
 
     def descr_get_doc(self, space):
         doc = self.rawfunctype.repr_fn_type(self.ffi, self.fnname)
         doc = '%s;\n\nCFFI C function from %s.lib' % (doc, self.modulename)
-        return space.wrap(doc)
+        return space.newtext(doc)
 
     def descr_get(self, space, w_obj, w_type=None):
         # never bind anything, but a __get__ is still present so that

From pypy.commits at gmail.com  Wed Nov  9 07:38:02 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:38:02 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: imp
Message-ID: <582318aa.12111c0a.f0f31.0351@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88262:ea22f9a7e6e7
Date: 2016-11-08 14:38 +0100
http://bitbucket.org/pypy/pypy/changeset/ea22f9a7e6e7/

Log:	imp

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -61,7 +61,7 @@
     verbose = space.sys.get_flag('verbose')
     if verbose >= level:
         w_stderr = space.sys.get('stderr')
-        space.call_method(w_stderr, "write", space.wrap(message))
+        space.call_method(w_stderr, "write", space.newtext(message))
 
 def file_exists(path):
     """Tests whether the given path is an existing regular file."""
@@ -152,7 +152,6 @@
     return result
 
 def _get_relative_name(space, modulename, level, w_globals):
-    w = space.wrap
     ctxt_w_package = space.finditem_str(w_globals, '__package__')
     ctxt_w_package = jit.promote(ctxt_w_package)
     level = jit.promote(level)
@@ -193,7 +192,7 @@
             else:
                 msg = ("Parent module '%s' not found while handling absolute "
                        "import" % ctxt_package)
-                space.warn(space.wrap(msg), space.w_RuntimeWarning)
+                space.warn(space.newtext(msg), space.w_RuntimeWarning)
 
         rel_modulename = ctxt_package[:dot_position]
         rel_level = rel_modulename.count('.') + 1
@@ -232,15 +231,15 @@
 
         if ctxt_w_path is not None:
             # __path__ is set, so __name__ is already the package name
-            space.setitem(w_globals, w("__package__"), ctxt_w_name)
+            space.setitem(w_globals, space.newtext("__package__"), ctxt_w_name)
         else:
             # Normal module, so work out the package name if any
             last_dot_position = ctxt_name.rfind('.')
             if last_dot_position < 0:
-                space.setitem(w_globals, w("__package__"), space.w_None)
+                space.setitem(w_globals, space.newtext("__package__"), space.w_None)
             else:
-                space.setitem(w_globals, w("__package__"),
-                              w(ctxt_name[:last_dot_position]))
+                space.setitem(w_globals, space.newtext("__package__"),
+                              space.newtext(ctxt_name[:last_dot_position]))
 
         if modulename:
             if rel_modulename:
@@ -257,7 +256,6 @@
     modulename = name
     if not modulename and level < 0:
         raise oefmt(space.w_ValueError, "Empty module name")
-    w = space.wrap
 
     if w_fromlist is not None and not space.is_true(w_fromlist):
         w_fromlist = None
@@ -291,7 +289,7 @@
 
     w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0)
     if rel_modulename is not None:
-        space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None)
+        space.setitem(space.sys.get('modules'), space.newtext(rel_modulename), space.w_None)
     return w_mod
 
 def absolute_import(space, modulename, baselevel, w_fromlist, tentative):
@@ -329,7 +327,7 @@
         w_mod = check_sys_modules_w(space, modulename)
         first = w_mod
         if w_fromlist is not None and w_mod is not None:
-            w_path = try_getattr(space, w_mod, space.wrap('__path__'))
+            w_path = try_getattr(space, w_mod, space.newtext('__path__'))
     else:
         level = 0
         first = None
@@ -344,7 +342,7 @@
             if level == baselevel:
                 first = w_mod
             if w_fromlist is not None:
-                w_path = try_getattr(space, w_mod, space.wrap('__path__'))
+                w_path = try_getattr(space, w_mod, space.newtext('__path__'))
             level += 1
     if w_fromlist is not None:
         # bit artificial code but important to not just unwrap w_fromlist
@@ -353,9 +351,9 @@
         if w_path is not None:
             length = space.len_w(w_fromlist)
             if length == 1 and space.eq_w(
-                    space.getitem(w_fromlist, space.wrap(0)),
-                    space.wrap('*')):
-                w_all = try_getattr(space, w_mod, space.wrap('__all__'))
+                    space.getitem(w_fromlist, space.newint(0)),
+                    space.newtext('*')):
+                w_all = try_getattr(space, w_mod, space.newtext('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
                     length = space.len_w(w_fromlist)
@@ -367,15 +365,13 @@
 
             if w_fromlist is not None:
                 for i in range(length):
-                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    w_name = space.getitem(w_fromlist, space.newint(i))
                     if try_getattr(space, w_mod, w_name) is None:
                         return None
         return w_mod
     return first
 
 def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
-    w = space.wrap
-
     if '/' in modulename or '\\' in modulename:
         raise oefmt(space.w_ImportError,
                     "Import by filename is not supported.")
@@ -398,16 +394,16 @@
             first = w_mod
             tentative = 0
         prefix.append(part)
-        w_path = try_getattr(space, w_mod, w('__path__'))
+        w_path = try_getattr(space, w_mod, space.newtext('__path__'))
         level += 1
 
     if w_fromlist is not None:
         if w_path is not None:
             length = space.len_w(w_fromlist)
             if length == 1 and space.eq_w(
-                    space.getitem(w_fromlist, space.wrap(0)),
-                    space.wrap('*')):
-                w_all = try_getattr(space, w_mod, w('__all__'))
+                    space.getitem(w_fromlist, space.newint(0)),
+                    space.newtext('*')):
+                w_all = try_getattr(space, w_mod, space.newtext('__all__'))
                 if w_all is not None:
                     w_fromlist = w_all
                     length = space.len_w(w_fromlist)
@@ -415,7 +411,7 @@
                     w_fromlist = None
             if w_fromlist is not None:
                 for i in range(length):
-                    w_name = space.getitem(w_fromlist, space.wrap(i))
+                    w_name = space.getitem(w_fromlist, space.newint(i))
                     if try_getattr(space, w_mod, w_name) is None:
                         load_part(space, w_path, prefix, space.str0_w(w_name),
                                   w_mod, tentative=1)
@@ -492,7 +488,7 @@
                 raise oefmt(space.w_ImportError, "existing directory")
 
     def find_module_w(self, space, __args__):
-        return space.wrap(None)
+        return space.w_None
 
 W_NullImporter.typedef = TypeDef(
     'imp.NullImporter',
@@ -567,7 +563,7 @@
                 else:
                     msg = ("Not importing directory '%s' missing __init__.py" %
                            (filepart,))
-                    space.warn(space.wrap(msg), space.w_ImportWarning)
+                    space.warn(space.newtext(msg), space.w_ImportWarning)
             modtype, suffix, filemode = find_modtype(space, filepart)
             try:
                 if modtype in (PY_SOURCE, PY_COMPILED, C_EXTENSION):
@@ -587,17 +583,16 @@
     return delayed_builtin
 
 def _prepare_module(space, w_mod, filename, pkgdir):
-    w = space.wrap
     space.sys.setmodule(w_mod)
-    space.setattr(w_mod, w('__file__'), space.wrap(filename))
-    space.setattr(w_mod, w('__doc__'), space.w_None)
+    space.setattr(w_mod, space.newtext('__file__'), space.newtext(filename))
+    space.setattr(w_mod, space.newtext('__doc__'), space.w_None)
     if pkgdir is not None:
-        space.setattr(w_mod, w('__path__'), space.newlist([w(pkgdir)]))
+        space.setattr(w_mod, space.newtext('__path__'), space.newlist([space.newtext(pkgdir)]))
 
 def add_module(space, w_name):
     w_mod = check_sys_modules(space, w_name)
     if w_mod is None:
-        w_mod = space.wrap(Module(space, w_name))
+        w_mod = Module(space, w_name)
         space.sys.setmodule(w_mod)
     return w_mod
 
@@ -634,7 +629,7 @@
                 if not oe.match(space, space.w_KeyError):
                     raise
         if w_mod is None:
-            w_mod = space.wrap(Module(space, w_modulename))
+            w_mod = Module(space, w_modulename)
         if find_info.modtype == PKG_DIRECTORY:
             pkgdir = find_info.filename
         else:
@@ -653,8 +648,8 @@
                 return load_compiled_module(space, w_modulename, w_mod, find_info.filename,
                                      magic, timestamp, find_info.stream.readall())
             elif find_info.modtype == PKG_DIRECTORY:
-                w_path = space.newlist([space.wrap(find_info.filename)])
-                space.setattr(w_mod, space.wrap('__path__'), w_path)
+                w_path = space.newlist([space.newtext(find_info.filename)])
+                space.setattr(w_mod, space.newtext('__path__'), w_path)
                 find_info = find_module(space, "__init__", None, "__init__",
                                         w_path, use_loader=False)
                 if find_info is None:
@@ -677,9 +672,8 @@
             raise
 
 def load_part(space, w_path, prefix, partname, w_parent, tentative):
-    w = space.wrap
     modulename = '.'.join(prefix + [partname])
-    w_modulename = w(modulename)
+    w_modulename = space.newtext(modulename)
     w_mod = check_sys_modules(space, w_modulename)
 
     if w_mod is not None:
@@ -693,7 +687,7 @@
             if find_info:
                 w_mod = load_module(space, w_modulename, find_info)
                 if w_parent is not None:
-                    space.setattr(w_parent, space.wrap(partname), w_mod)
+                    space.setattr(w_parent, space.newtext(partname), w_mod)
                 return w_mod
         finally:
             if find_info:
@@ -717,7 +711,7 @@
     if not space.is_w(space.type(w_module), space.type(space.sys)):
         raise oefmt(space.w_TypeError, "reload() argument must be module")
 
-    w_modulename = space.getattr(w_module, space.wrap("__name__"))
+    w_modulename = space.getattr(w_module, space.newtext("__name__"))
     modulename = space.str0_w(w_modulename)
     if not space.is_w(check_sys_modules(space, w_modulename), w_module):
         raise oefmt(space.w_ImportError,
@@ -741,7 +735,7 @@
                 raise oefmt(space.w_ImportError,
                             "reload(): parent %s not in sys.modules",
                             parent_name)
-            w_path = space.getattr(w_parent, space.wrap("__path__"))
+            w_path = space.getattr(w_parent, space.newtext("__path__"))
         else:
             w_path = None
 
@@ -888,10 +882,10 @@
     Execute a code object in the module's dict.  Returns
     'sys.modules[modulename]', which must exist.
     """
-    w_dict = space.getattr(w_mod, space.wrap('__dict__'))
+    w_dict = space.getattr(w_mod, space.newtext('__dict__'))
     space.call_method(w_dict, 'setdefault',
-                      space.wrap('__builtins__'),
-                      space.wrap(space.builtin))
+                      space.newtext('__builtins__'),
+                      space.builtin)
     code_w.exec_code(space, w_dict, w_dict)
 
     if check_afterwards:
@@ -910,7 +904,6 @@
     Load a source module from a given file.  Returns the result
     of sys.modules[modulename], which must exist.
     """
-    w = space.wrap
 
     log_pyverbose(space, 1, "import %s # from %s\n" %
                   (space.str_w(w_modulename), pathname))
@@ -930,7 +923,7 @@
                 stream.close()
             except StreamErrors:
                 pass
-        space.setattr(w_mod, w('__file__'), w(cpathname))
+        space.setattr(w_mod, space.newtext('__file__'), space.newtext(cpathname))
     else:
         code_w = parse_source_module(space, pathname, source)
 
@@ -1024,7 +1017,7 @@
     """ Read a code object from a file and check it for validity """
 
     w_marshal = space.getbuiltinmodule('marshal')
-    w_code = space.call_method(w_marshal, 'loads', space.wrap(strbuf))
+    w_code = space.call_method(w_marshal, 'loads', space.newbytes(strbuf))
     if not isinstance(w_code, Code):
         raise oefmt(space.w_ImportError, "Non-code object in %s", cpathname)
     return w_code
@@ -1074,8 +1067,8 @@
     """
     w_marshal = space.getbuiltinmodule('marshal')
     try:
-        w_str = space.call_method(w_marshal, 'dumps', space.wrap(co),
-                                  space.wrap(MARSHAL_VERSION_FOR_PYC))
+        w_str = space.call_method(w_marshal, 'dumps', co,
+                                  space.newint(MARSHAL_VERSION_FOR_PYC))
         strbuf = space.str_w(w_str)
     except OperationError as e:
         if e.async(space):
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -58,7 +58,7 @@
     if not find_info:
         raise oefmt(space.w_ImportError, "No module named %s", name)
 
-    w_filename = space.wrap(find_info.filename)
+    w_filename = space.newtext(find_info.filename)
     stream = find_info.stream
 
     if stream is not None:
@@ -66,13 +66,13 @@
         fileobj.fdopenstream(
             stream, stream.try_to_find_file_descriptor(),
             find_info.filemode, w_filename)
-        w_fileobj = space.wrap(fileobj)
+        w_fileobj = fileobj
     else:
         w_fileobj = space.w_None
     w_import_info = space.newtuple(
-        [space.wrap(find_info.suffix),
-         space.wrap(find_info.filemode),
-         space.wrap(find_info.modtype)])
+        [space.newtext(find_info.suffix),
+         space.newtext(find_info.filemode),
+         space.newint(find_info.modtype)])
     return space.newtuple([w_fileobj, w_filename, w_import_info])
 
 def load_module(space, w_name, w_file, w_filename, w_info):
@@ -99,7 +99,7 @@
 
     stream = get_file(space, w_file, filename, 'U')
 
-    w_mod = space.wrap(Module(space, w_modulename))
+    w_mod = Module(space, w_modulename)
     importing._prepare_module(space, w_mod, filename, None)
 
     w_mod = importing.load_source_module(
@@ -127,7 +127,7 @@
 
 @unwrap_spec(filename='str0')
 def load_compiled(space, w_modulename, filename, w_file=None):
-    w_mod = space.wrap(Module(space, w_modulename))
+    w_mod = Module(space, w_modulename)
     importing._prepare_module(space, w_mod, filename, None)
     return _run_compiled_module(space, w_modulename, filename, w_file, w_mod,
                                 check_afterwards=True)
@@ -140,7 +140,7 @@
     return importing.check_sys_modules(space, w_modulename)
 
 def new_module(space, w_name):
-    return space.wrap(Module(space, w_name, add_package=False))
+    return Module(space, w_name, add_package=False)
 
 def init_builtin(space, w_name):
     name = space.str0_w(w_name)
@@ -157,10 +157,10 @@
 def is_builtin(space, w_name):
     name = space.str0_w(w_name)
     if name not in space.builtin_modules:
-        return space.wrap(0)
+        return space.newint(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:
-        return space.wrap(-1)   # cannot be initialized again
-    return space.wrap(1)
+        return space.newint(-1)   # cannot be initialized again
+    return space.newint(1)
 
 def is_frozen(space, w_name):
     return space.w_False
@@ -169,7 +169,7 @@
 
 def lock_held(space):
     if space.config.objspace.usemodules.thread:
-        return space.wrap(importing.getimportlock(space).lock_held_by_anyone())
+        return space.newbool(importing.getimportlock(space).lock_held_by_anyone())
     else:
         return space.w_False
 

From pypy.commits at gmail.com  Wed Nov  9 07:38:04 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 04:38:04 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: parser
Message-ID: <582318ac.c1a5c20a.a8a85.994c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88263:c5c3e2fb39a6
Date: 2016-11-08 15:08 +0100
http://bitbucket.org/pypy/pypy/changeset/c5c3e2fb39a6/

Log:	parser

diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py
--- a/pypy/module/parser/pyparser.py
+++ b/pypy/module/parser/pyparser.py
@@ -17,26 +17,26 @@
     def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column):
         if node.num_children():
             seq_w = [None]*(node.num_children() + 1)
-            seq_w[0] = space.wrap(node.type)
+            seq_w[0] = space.newint(node.type)
             for i in range(1, node.num_children() + 1):
                 seq_w[i] = self._build_app_tree(space, node.get_child(i - 1),
                                                 seq_maker, with_lineno,
                                                 with_column)
         else:
             seq_w = [None]*(2 + with_lineno + with_column)
-            seq_w[0] = space.wrap(node.type)
-            seq_w[1] = space.wrap(node.get_value())
+            seq_w[0] = space.newint(node.type)
+            seq_w[1] = space.newtext(node.get_value())
             if with_lineno:
-                seq_w[2] = space.wrap(node.get_lineno())
+                seq_w[2] = space.newint(node.get_lineno())
             if with_column:
-                seq_w[3] = space.wrap(node.get_column())
+                seq_w[3] = space.newint(node.get_column())
         return seq_maker(seq_w)
 
     def descr_issuite(self, space):
-        return space.wrap(self.tree.type == pygram.syms.file_input)
+        return space.newbool(self.tree.type == pygram.syms.file_input)
 
     def descr_isexpr(self, space):
-        return space.wrap(self.tree.type == pygram.syms.eval_input)
+        return space.newbool(self.tree.type == pygram.syms.eval_input)
 
     @unwrap_spec(line_info=bool, col_info=bool)
     def descr_totuple(self, space, line_info=False, col_info=False):
@@ -60,7 +60,7 @@
         except error.SyntaxError as e:
             raise OperationError(space.w_SyntaxError,
                                  e.wrap_info(space))
-        return space.wrap(result)
+        return result
 
 W_STType.typedef = TypeDef("parser.st",
     issuite=interp2app(W_STType.descr_issuite),
@@ -82,7 +82,7 @@
     except error.SyntaxError as e:
         raise OperationError(space.w_SyntaxError,
                              e.wrap_info(space))
-    return space.wrap(W_STType(tree, mode))
+    return W_STType(tree, mode)
 
 
 @unwrap_spec(source=str)
@@ -105,12 +105,12 @@
 
 @unwrap_spec(w_st=W_STType)
 def st2tuple(space, w_st, __args__):
-    return space.call_args(space.getattr(w_st, space.wrap("totuple")), __args__)
+    return space.call_args(space.getattr(w_st, space.newtext("totuple")), __args__)
 
 @unwrap_spec(w_st=W_STType)
 def st2list(space, w_st, __args__):
-    return space.call_args(space.getattr(w_st, space.wrap("tolist")), __args__)
+    return space.call_args(space.getattr(w_st, space.newtext("tolist")), __args__)
 
 @unwrap_spec(w_st=W_STType)
 def compilest(space, w_st, __args__):
-    return space.call_args(space.getattr(w_st, space.wrap("compile")), __args__)
+    return space.call_args(space.getattr(w_st, space.newtext("compile")), __args__)

From pypy.commits at gmail.com  Wed Nov  9 09:06:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 06:06:07 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: mark some vector tests x86
 only (test cases are written for x86 ISA assumptions)
Message-ID: <58232d4f.83f8c20a.ac90e.b5cf@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r88264:aff251e54385
Date: 2016-11-08 09:52 +0100
http://bitbucket.org/pypy/pypy/changeset/aff251e54385/

Log:	mark some vector tests x86 only (test cases are written for x86 ISA
	assumptions)

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -418,9 +418,7 @@
     def test_sum_float_to_int16(self):
         result = self.run("sum_float_to_int16")
         assert result == sum(range(30))
-        # one can argue that this is not desired,
-        # but unpacking exactly hits savings = 0
-        self.check_vectorized(1, 1)
+
     def define_sum_float_to_int32():
         return """
         a = |30|
@@ -429,7 +427,6 @@
     def test_sum_float_to_int32(self):
         result = self.run("sum_float_to_int32")
         assert result == sum(range(30))
-        self.check_vectorized(1, 1)
 
     def define_sum_float_to_float32():
         return """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -1,6 +1,7 @@
 import py
 import sys
 import pytest
+import platform
 
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
@@ -125,6 +126,7 @@
         """, False)
         self.assert_equal(loop2, loop3)
 
+    @py.test.mark.skipif("not platform.machine().startswith('x86')")
     def test_int_to_float(self):
         loop1 = self.parse_trace("""
         i10 = raw_load_i(p0, i0, descr=long)
@@ -227,6 +229,7 @@
         """, False, additional_args=['v10[2xi64]'])
         self.assert_equal(loop2, loop3)
 
+    @py.test.mark.skipif("not platform.machine().startswith('x86')")
     def test_cast_float_to_int(self):
         loop1 = self.parse_trace("""
         f10 = raw_load_f(p0, i1, descr=double)
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -3,7 +3,7 @@
 import pytest
 import math
 import functools
-from hypothesis import given, note, strategies as st
+from hypothesis import given, note, strategies as st, settings
 from rpython.jit.metainterp.warmspot import ll_meta_interp, get_stats
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.jit.codewriter.policy import StopAtXPolicy
@@ -150,9 +150,7 @@
                 i += size
 
         la = data.draw(st.lists(st.floats(), min_size=10, max_size=150))
-        #la = [0.0, 0.0, 0.0, 0.0, 5e-324, 0.0, 0.0, 5e-324, 0.0, 0.0]
         l = len(la)
-        #lb = [0.0] * l
         lb = data.draw(st.lists(st.floats(), min_size=l, max_size=l))
 
         rawstorage = RawStorage()
@@ -181,15 +179,14 @@
         vec_float_binary(lambda a,b: a-b, rffi.DOUBLE)
     test_vec_float_mul = \
         vec_float_binary(lambda a,b: a*b, rffi.DOUBLE)
-    #test_vec_float_div = \
-    #    vec_float_binary(lambda a,b: a/b, rffi.DOUBLE)
 
     test_vec_float_cmp_eq = \
         vec_float_binary(lambda a,b: a == b, rffi.DOUBLE)
     test_vec_float_cmp_ne = \
         vec_float_binary(lambda a,b: a != b, rffi.DOUBLE)
 
-    def _vector_simple_int(self, func, type, strat, data):
+    def _vector_simple_int(self, func, type, la):
+        oldfunc = func
         func = always_inline(func)
 
         size = rffi.sizeof(type)
@@ -204,9 +201,8 @@
                 raw_storage_setitem(vc, i, rffi.cast(type,c))
                 i += size
 
-        la = data.draw(st.lists(strat, min_size=10, max_size=150))
         l = len(la)
-        lb = data.draw(st.lists(strat, min_size=l, max_size=l))
+        lb = list(reversed(la))[:]
 
         rawstorage = RawStorage()
         va = rawstorage.new(la, type)
@@ -216,7 +212,7 @@
 
         for i in range(l):
             c = raw_storage_getitem(type,vc,i*size)
-            assert rffi.cast(type, func(la[i], lb[i])) == c
+            assert rffi.cast(type, oldfunc(la[i], lb[i])) == c
 
         rawstorage.clear()
 
@@ -225,50 +221,52 @@
         bits = size*8
         assert 0 <= bits <= 64
         integers = st.integers(min_value=-2**(bits-1), max_value=2**(bits-1)-1)
-        return pytest.mark.parametrize('func,type,strat', [
-            (arith_func, type, integers)
-        ])(given(data=st.data())(test_func))
+        @given(st.lists(integers, min_size=10, max_size=15))
+        @settings(max_examples=20)
+        def tf(self, la):
+            return test_func(self, arith_func, type, la)
+        return tf
 
     vec_int_arith = functools.partial(vec_int_arith, _vector_simple_int)
 
-    test_vec_signed_add = \
+    test_vec_simple_int_signed_add = \
         vec_int_arith(lambda a,b: intmask(a+b), rffi.SIGNED)
-    test_vec_int_add = \
-        vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.INT)
-    test_vec_short_add = \
-        vec_int_arith(lambda a,b: r_int(a)+r_int(b), rffi.SHORT)
+    test_vec_simple_int_int_add = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)+r_int(b)), rffi.INT)
+    test_vec_simple_int_short_add = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)+r_int(b)), rffi.SHORT)
 
-    test_vec_signed_sub = \
+    test_vec_simple_int_signed_sub = \
         vec_int_arith(lambda a,b: intmask(a-b), rffi.SIGNED)
-    test_vec_sub_int = \
-        vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.INT)
-    test_vec_sub_short = \
-        vec_int_arith(lambda a,b: r_int(a)-r_int(b), rffi.SHORT)
+    test_vec_simple_int_sub_int = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)-r_int(b)), rffi.INT)
+    test_vec_simple_int_sub_short = \
+        vec_int_arith(lambda a,b: intmask(r_int(a)-r_int(b)), rffi.SHORT)
 
-    test_vec_signed_and = \
+    test_vec_simple_int_signed_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SIGNED)
-    test_vec_int_and = \
+    test_vec_simple_int_int_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.INT)
-    test_vec_short_and = \
+    test_vec_simple_int_short_and = \
         vec_int_arith(lambda a,b: intmask(a)&intmask(b), rffi.SHORT)
 
-    test_vec_or_signed = \
+    test_vec_simple_int_or_signed = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SIGNED)
-    test_vec_or_int = \
+    test_vec_simple_int_or_int = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.INT)
-    test_vec_or_short = \
+    test_vec_simple_int_or_short = \
         vec_int_arith(lambda a,b: intmask(a)|intmask(b), rffi.SHORT)
 
-    test_vec_xor_signed = \
+    test_vec_simple_int_xor_signed = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SIGNED)
-    test_vec_xor_int = \
+    test_vec_simple_int_xor_int = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.INT)
-    test_vec_xor_short = \
+    test_vec_simple_int_xor_short = \
         vec_int_arith(lambda a,b: intmask(a)^intmask(b), rffi.SHORT)
 
-    test_vec_int_cmp_eq = \
+    test_vec_simple_int_int_cmp_eq = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
-    test_vec_int_cmp_ne = \
+    test_vec_simple_int_int_cmp_ne = \
         vec_int_arith(lambda a,b: a == b, rffi.SIGNED)
 
     @py.test.mark.parametrize('i',[1,2,3,4,9])
@@ -414,9 +412,9 @@
 
     test_vec_int_sum = vec_reduce(st.integers(min_value=-2**(64-1), max_value=2**(64-1)-1),
                              lambda a,b: lltype.intmask(lltype.intmask(a)+lltype.intmask(b)), lltype.Signed)
-    test_vec_float_sum = vec_reduce(st.floats(), lambda a,b: a+b, rffi.DOUBLE)
-    test_vec_float_prod = vec_reduce(st.floats(min_value=-100, max_value=100,
-                                               allow_nan=False, allow_infinity=False), lambda a,b: a*b, rffi.DOUBLE)
+    small_floats = st.floats(min_value=-100, max_value=100, allow_nan=False, allow_infinity=False)
+    test_vec_float_sum = vec_reduce(small_floats, lambda a,b: a+b, rffi.DOUBLE)
+    test_vec_float_prod = vec_reduce(small_floats, lambda a,b: a*b, rffi.DOUBLE)
 
 
     def test_constant_expand(self):

From pypy.commits at gmail.com  Wed Nov  9 10:33:24 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: move file from imp/test to interpreter,
 as basis of more fsencode tests
Message-ID: <582341c4.e626c20a.c4c66.09b9@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88265:e61a88da0023
Date: 2016-11-09 13:35 +0100
http://bitbucket.org/pypy/pypy/changeset/e61a88da0023/

Log:	move file from imp/test to interpreter, as basis of more fsencode
	tests

diff --git a/pypy/module/imp/test/support.py b/pypy/interpreter/test/test_fsencode.py
rename from pypy/module/imp/test/support.py
rename to pypy/interpreter/test/test_fsencode.py
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -4,7 +4,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pycode import PyCode
-from pypy.module.imp.test.support import BaseImportTest
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
 from rpython.tool.udir import udir
 from rpython.rlib import streamio
 from pypy.tool.option import make_config
@@ -177,13 +177,13 @@
     """)
 
 
-class AppTestImport(BaseImportTest):
+class AppTestImport(BaseFSEncodeTest):
     spaceconfig = {
         "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
     }
 
     def setup_class(cls):
-        BaseImportTest.setup_class.im_func(cls)
+        BaseFSEncodeTest.setup_class.im_func(cls)
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.w_saved_modules = _setup(cls)
         #XXX Compile class

From pypy.commits at gmail.com  Wed Nov  9 10:33:26 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add some simple fs en/decode tests
Message-ID: <582341c6.c3181c0a.91181.81de@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88266:c9b8dd4d8872
Date: 2016-11-09 14:17 +0100
http://bitbucket.org/pypy/pypy/changeset/c9b8dd4d8872/

Log:	add some simple fs en/decode tests

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1068,9 +1068,6 @@
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
 
-    def wrap_fsdecoded(self, x):
-        return self.fsdecode(self.newbytes(x))
-
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
@@ -1680,6 +1677,9 @@
             w_obj = self.fsdecode(w_obj)
         return self.unicode0_w(w_obj)
 
+    def wrap_fsdecoded(self, x):
+        return self.fsdecode(self.newbytes(x))
+
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
         # For cases where you need to accept bools and ints and nothing
diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -1,6 +1,7 @@
 import sys
+from pypy.interpreter.error import OperationError
 
-class BaseImportTest:
+class BaseFSEncodeTest:
 
     def setup_class(cls):
         space = cls.space
@@ -60,3 +61,24 @@
 
     if special_char:
         return special_char.decode(fsenc)
+
+class TestFSEncode(BaseFSEncodeTest):
+    def test_fsencode_fsdecode(self):
+        space = self.space
+        strs = [u"/home/bar/baz", u"c:\\"]
+        if self.special_char:
+            strs.append(self.special_char)
+        for st in strs:
+            # check roundtrip
+            w_st = space.newunicode(st)
+            w_enc = space.fsencode(w_st)
+            w_st2 = space.fsdecode(w_enc)
+            assert space.eq_w(w_st, w_st2)
+            assert space.fsdecode_w(w_enc) == st
+
+            assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
+            assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
+
+    def test_error(self):
+        raises(OperationError, self.space.fsencode, self.space.newbytes(self.testfn_unencodable))
+

From pypy.commits at gmail.com  Wed Nov  9 10:33:29 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: use fsencode in _ssl module.
Message-ID: <582341c9.8a29c20a.26431.09ec@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88268:1eca675443fa
Date: 2016-11-09 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/1eca675443fa/

Log:	use fsencode in _ssl module.

	(the test is not actually good. it passes before the change too)

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1597,7 +1597,7 @@
             libssl_SSL_CTX_set_default_passwd_cb_userdata(
                 self.ctx, None)
 
-    @unwrap_spec(filepath=str)
+    @unwrap_spec(filepath='fsencode')
     def load_dh_params_w(self, space, filepath):
         bio = libssl_BIO_new_file(filepath, "r")
         if not bio:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -1,6 +1,8 @@
 from rpython.tool.udir import udir
 import os
 
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
+
 class AppTestSSL:
     spaceconfig = dict(usemodules=('_ssl', '_socket', 'select', 'struct',
                                    'binascii', 'thread'))
@@ -341,10 +343,11 @@
             """)
 
 
-class AppTestContext:
+class AppTestContext(BaseFSEncodeTest):
     spaceconfig = dict(usemodules=('_ssl',))
 
     def setup_class(cls):
+        BaseFSEncodeTest.setup_class.im_func(cls)
         tmpfile = udir / "tmpfile.pem"
         tmpfile.write(SSL_CERTIFICATE + SSL_PRIVATE_KEY)
         cls.w_keycert = cls.space.wrap(str(tmpfile))
@@ -366,8 +369,14 @@
         tmpfile = udir / "python.org.pem"
         tmpfile.write(SVN_PYTHON_ORG_ROOT_CERT)
         cls.w_python_org_cert = cls.space.wrap(str(tmpfile))
-        cls.w_dh512 = cls.space.wrap(os.path.join(
-            os.path.dirname(__file__), 'dh512.pem'))
+        tmpfile = udir / cls.special_char
+        fn = os.path.join(
+            os.path.dirname(__file__), 'dh512.pem')
+        with file(fn) as f:
+            s = f.read()
+        tmpfile.write(s)
+        cls.w_dh512 = cls.space.wrap(fn)
+        cls.w_dh512special = cls.space.wrap(str(tmpfile))
 
     def test_load_cert_chain(self):
         import _ssl, errno
@@ -443,6 +452,9 @@
         exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
         assert exc.value.errno == errno.ENOENT
 
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.load_dh_params(self.dh512special)
+
     def test_set_ecdh_curve(self):
         import _ssl
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)

From pypy.commits at gmail.com  Wed Nov  9 10:33:31 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: more uses of fs en/decoding
Message-ID: <582341cb.87b2c20a.45762.0b3b@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88269:54e6492211f6
Date: 2016-11-09 16:30 +0100
http://bitbucket.org/pypy/pypy/changeset/54e6492211f6/

Log:	more uses of fs en/decoding

	(no tests. I haven't actually found a way to test this on Linux)

diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -42,7 +42,7 @@
     w_name = space.wrap(name)
     module = Module(space, w_name)
     if path is not None:
-        module.setdictvalue(space, '__file__', space.wrap(path))
+        module.setdictvalue(space, '__file__', space.wrap_fsdecoded(path))
     module.setdictvalue(space, 'ffi', space.wrap(ffi))
     module.setdictvalue(space, 'lib', space.wrap(lib))
     w_modules_dict = space.sys.get('modules')
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1511,7 +1511,7 @@
     from pypy.module._cffi_backend import cffi1_module
     cffi1_module.load_cffi1_module(space, name, path, initptr)
 
- at unwrap_spec(path=str, name=str)
+ at unwrap_spec(path='fsencode', name=str)
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
     # modules (cpyext) and to load CFFI-style extension modules
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -47,7 +47,7 @@
     state.package_context = None, None
 
     if f_path is not None:
-        dict_w = {'__file__': space.wrap(f_path)}
+        dict_w = {'__file__': space.wrap_fsdecoded(f_path)}
     else:
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_mod, modname)

From pypy.commits at gmail.com  Wed Nov  9 10:33:33 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge
Message-ID: <582341cd.88051c0a.314c.84e6@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88270:ecf5ae8f6cbc
Date: 2016-11-09 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ecf5ae8f6cbc/

Log:	merge

diff --git a/lib-python/3/test/test_httpservers.py b/lib-python/3/test/test_httpservers.py
--- a/lib-python/3/test/test_httpservers.py
+++ b/lib-python/3/test/test_httpservers.py
@@ -254,7 +254,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('GET', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         self.assertTrue(
             err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n'))
@@ -265,7 +266,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('ERROR', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         lines = err.getvalue().split('\n')
         self.assertTrue(lines[0].endswith('code 404, message File not found'))

From pypy.commits at gmail.com  Wed Nov  9 10:33:28 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 09 Nov 2016 07:33:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: misunderstood the use of
 testfn_unencodable, so this doesn't make sense
Message-ID: <582341c8.94841c0a.ed79.832d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88267:ba1e8995a96f
Date: 2016-11-09 16:19 +0100
http://bitbucket.org/pypy/pypy/changeset/ba1e8995a96f/

Log:	misunderstood the use of testfn_unencodable, so this doesn't make
	sense

diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -78,7 +78,3 @@
 
             assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
             assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
-
-    def test_error(self):
-        raises(OperationError, self.space.fsencode, self.space.newbytes(self.testfn_unencodable))
-

From pypy.commits at gmail.com  Wed Nov  9 10:50:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:50:22 -0800 (PST)
Subject: [pypy-commit] cffi default: Fix a warning for PyPy3
Message-ID: <582345be.471ec20a.2f6d8.15c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2808:0ecb431b2b94
Date: 2016-11-09 16:21 +0100
http://bitbucket.org/cffi/cffi/changeset/0ecb431b2b94/

Log:	Fix a warning for PyPy3

diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('PyMODINIT_FUNC')
+        prnt('void')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:

From pypy.commits at gmail.com  Wed Nov  9 10:50:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:50:54 -0800 (PST)
Subject: [pypy-commit] pypy default: Move patch_sys() from _cffi_backend to
 pypy_setup_home() calling code
Message-ID: <582345de.120f1c0a.50b03.4dc4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88271:440d820c878f
Date: 2016-11-09 14:58 +0100
http://bitbucket.org/pypy/pypy/changeset/440d820c878f/

Log:	Move patch_sys() from _cffi_backend to pypy_setup_home() calling
	code from app_main.py. This is mostly motivated by py3.5, where
	setting up correctly 'sys.std*' is more involved.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -33,7 +33,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -103,18 +108,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -122,6 +132,9 @@
                 debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -275,6 +275,10 @@
 def set_fully_buffered_io():
     sys.stdout = sys.__stdout__ = fdopen(1, 'w')
 
+def initstdio(unbuffered=False):
+    if unbuffered:
+        set_unbuffered_io()
+
 # ____________________________________________________________
 # Main entry point
 
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -20,7 +20,7 @@
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    lls = rffi.str2charp("sys")
+    lls = rffi.str2charp("sys   # should give a NameError")
     execute_source(lls)
     lltype.free(lls, flavor='raw')
     # did not crash - the same globals
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,6 @@
     def startup(self, space):
         from pypy.module._cffi_backend import embedding
         embedding.glob.space = space
-        embedding.glob.patched_sys = False
 
 
 def get_dict_rtld_constants():
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -45,25 +45,6 @@
     pass
 glob = Global()
 
-def patch_sys(space):
-    # Annoying: CPython would just use the C-level std{in,out,err} as
-    # configured by the main application, for example in binary mode
-    # on Windows or with buffering turned off.  We can't easily do the
-    # same.  Instead, go for the safest bet (but possibly bad for
-    # performance) and open sys.std{in,out,err} unbuffered.  On
-    # Windows I guess binary mode is a better default choice.
-    #
-    # XXX if needed, we could add support for a flag passed to
-    # pypy_init_embedded_cffi_module().
-    if not glob.patched_sys:
-        space.appexec([], """():
-            import os, sys
-            sys.stdin  = sys.__stdin__  = os.fdopen(0, 'rb', 0)
-            sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0)
-            sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0)
-        """)
-        glob.patched_sys = True
-
 
 def pypy_init_embedded_cffi_module(version, init_struct):
     # called from __init__.py
@@ -76,7 +57,6 @@
         must_leave = False
         try:
             must_leave = space.threadlocals.try_enter_thread(space)
-            patch_sys(space)
             load_embedded_cffi_module(space, version, init_struct)
             res = 0
         except OperationError as operr:

From pypy.commits at gmail.com  Wed Nov  9 10:50:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:50:58 -0800 (PST)
Subject: [pypy-commit] pypy default: import cffi/0ecb431b2b94
Message-ID: <582345e2.44941c0a.9f413.4ca2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88273:1cf0e7cfa5ef
Date: 2016-11-09 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/1cf0e7cfa5ef/

Log:	import cffi/0ecb431b2b94

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('PyMODINIT_FUNC')
+        prnt('void')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:

From pypy.commits at gmail.com  Wed Nov  9 10:51:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:51:02 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <582345e6.ca13c20a.ba6f7.0d8b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88275:9d3789ff54e1
Date: 2016-11-09 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/9d3789ff54e1/

Log:	merge heads

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,7 +1,7 @@
 # Edit these appropriately before running this script
 maj=5
-min=4
-rev=1
+min=6
+rev=0
 branchname=release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -76,7 +76,6 @@
     'CLGRJ':   ('rie_b',  ['\xEC','\x65']),
     # compare and swap
     'CSG':     ('rsy_a', ['\xEB','\x30']),
-
 }
 
 logic_mnemonic_codes = {
@@ -340,7 +339,7 @@
     'VLVG':  ('vrs_b', ['\xE7','\x22']),
     'VLGV':  ('vrs_c', ['\xE7','\x21']),
 
-    # '': ('', ['','']),
+    'VSEG':  ('vrr_a', ['\xE7','\x5F'], 'v,v,m'),
 }
 
 all_mnemonic_codes.update(arith_mnemonic_codes)
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -139,6 +139,8 @@
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(0), l.itemsize_to_mask(nsize))
             self.mc.VLGV(r.SCRATCH, loc0, l.addr(1), l.itemsize_to_mask(osize))
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(1), l.itemsize_to_mask(nsize))
+            if nsize == 8:
+                self.mc.VSEG(resloc, resloc, l.itemsize_to_mask(osize))
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -14,6 +14,7 @@
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
 
 
 def forwarded_vecinfo(op):
@@ -315,11 +316,8 @@
 
 def failnbail_transformation(msg):
     msg = '%s\n' % msg
-    if we_are_translated():
-        llop.debug_print(lltype.Void, msg)
-    else:
-        import pdb; pdb.set_trace()
-    raise NotImplementedError(msg)
+    debug_print(msg)
+    raise NotAVectorizeableLoop
 
 def turn_into_vector(state, pack):
     """ Turn a pack into a vector instruction """
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -860,6 +860,37 @@
         free_raw_storage(vb)
         free_raw_storage(vc)
 
+    def test_float_int32_casts(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            j = 0
+            while j < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(rffi.DOUBLE,va,j)
+                b = raw_storage_getitem(rffi.INT,vb,i)
+                c = a+rffi.cast(rffi.DOUBLE,b)
+                raw_storage_setitem(vc, j, c)
+                i += 4
+                j += 8
+
+        count = 32
+        va = alloc_raw_storage(8*count, zero=True)
+        vb = alloc_raw_storage(4*count, zero=True)
+        for i,v in enumerate([1.0,2.0,3.0,4.0]*(count/4)):
+            raw_storage_setitem(va, i*8, rffi.cast(rffi.DOUBLE,v))
+        for i,v in enumerate([-1,-2,-3,-4]*(count/4)):
+            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
+
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
+
+        free_raw_storage(va)
+        free_raw_storage(vb)
+        free_raw_storage(vc)
+
 
 class TestLLtype(LLJitMixin, VectorizeTests):
     # skip some tests on this backend

From pypy.commits at gmail.com  Wed Nov  9 10:50:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:50:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582345e0.43e61c0a.3cdac.952a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88272:5980a1419dc2
Date: 2016-11-09 15:04 +0100
http://bitbucket.org/pypy/pypy/changeset/5980a1419dc2/

Log:	hg merge default

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -34,7 +34,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -110,18 +115,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('builtins'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -129,6 +139,9 @@
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -20,7 +20,7 @@
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    lls = rffi.str2charp("sys")
+    lls = rffi.str2charp("sys   # should give a NameError")
     execute_source(lls)
     lltype.free(lls, flavor='raw')
     # did not crash - the same globals
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,6 @@
     def startup(self, space):
         from pypy.module._cffi_backend import embedding
         embedding.glob.space = space
-        embedding.glob.patched_sys = False
 
 
 def get_dict_rtld_constants():
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -45,25 +45,6 @@
     pass
 glob = Global()
 
-def patch_sys(space):
-    # Annoying: CPython would just use the C-level std{in,out,err} as
-    # configured by the main application, for example in binary mode
-    # on Windows or with buffering turned off.  We can't easily do the
-    # same.  Instead, go for the safest bet (but possibly bad for
-    # performance) and open sys.std{in,out,err} unbuffered.  On
-    # Windows I guess binary mode is a better default choice.
-    #
-    # XXX if needed, we could add support for a flag passed to
-    # pypy_init_embedded_cffi_module().
-    if not glob.patched_sys:
-        space.appexec([], """():
-            import os, sys
-            sys.stdin  = sys.__stdin__  = os.fdopen(0, 'rb', 0)
-            sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0)
-            sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0)
-        """)
-        glob.patched_sys = True
-
 
 def pypy_init_embedded_cffi_module(version, init_struct):
     # called from __init__.py
@@ -76,7 +57,6 @@
         must_leave = False
         try:
             must_leave = space.threadlocals.try_enter_thread(space)
-            patch_sys(space)
             load_embedded_cffi_module(space, version, init_struct)
             res = 0
         except OperationError as operr:
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -18,7 +18,7 @@
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
-from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
 from rpython.rlib.objectmodel import Symbolic
 
 class LLAsmInfo(object):
@@ -840,6 +840,11 @@
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
         assert len(vx) == len(vy) == count
+        return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
+    """
+    vector_float_arith_code = """
+    def bh_vec_{0}_{1}(self, vx, vy, count):
+        assert len(vx) == len(vy) == count
         return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
     """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
@@ -847,11 +852,12 @@
     exec py.code.Source(vector_arith_code.format('int','mul','*')).compile()
     exec py.code.Source(vector_arith_code.format('int','and','&')).compile()
     exec py.code.Source(vector_arith_code.format('int','or','|')).compile()
-    exec py.code.Source(vector_arith_code.format('float','add','+')).compile()
-    exec py.code.Source(vector_arith_code.format('float','sub','-')).compile()
-    exec py.code.Source(vector_arith_code.format('float','mul','*')).compile()
-    exec py.code.Source(vector_arith_code.format('float','truediv','/')).compile()
-    exec py.code.Source(vector_arith_code.format('float','eq','==')).compile()
+
+    exec py.code.Source(vector_float_arith_code.format('float','add','+')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','sub','-')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','mul','*')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','truediv','/')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','eq','==')).compile()
 
     def bh_vec_float_neg(self, vx, count):
         return [e * -1 for e in vx]
@@ -891,9 +897,6 @@
         return [longlong.getfloatstorage(float(longlong.int2singlefloat(v)))
                 for v in vx]
 
-        a = float(a)
-        return longlong.getfloatstorage(a)
-
     def bh_vec_cast_float_to_int(self, vx, count):
         return [int(x) for x in vx]
 
@@ -934,27 +937,26 @@
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
     def build_load(func):
-        def method(self, struct, offset, scale, disp, descr, _count):
+        def load(self, struct, offset, scale, disp, descr, _count):
             values = []
             count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
-            adr = struct + (offset * scale + disp)
+            adr = support.addr_add_bytes(struct, (offset * scale + disp))
             a = support.cast_arg(lltype.Ptr(descr.A), adr)
             array = a._obj
             for i in range(count):
                 val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
-        return method
+        return load
 
     bh_vec_load_i = build_load(bh_getarrayitem_raw)
     bh_vec_load_f = build_load(bh_getarrayitem_raw)
     del build_load
 
     def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
-        stride = descr.get_item_size_in_bytes()
-        adr = struct + (offset * scale + disp)
+        adr = support.addr_add_bytes(struct, offset * scale + disp)
         a = support.cast_arg(lltype.Ptr(descr.A), adr)
         array = a._obj
         for i,n in enumerate(newvalues):
@@ -1557,7 +1559,7 @@
             if opname.startswith('vec_'):
                 # pre vector op
                 count = self.current_op.count
-                assert count >= 1
+                assert count >= 0
                 new_args = new_args + (count,)
             result = getattr(self.cpu, 'bh_' + opname)(*new_args)
             if isinstance(result, list):
diff --git a/rpython/jit/backend/llgraph/support.py b/rpython/jit/backend/llgraph/support.py
--- a/rpython/jit/backend/llgraph/support.py
+++ b/rpython/jit/backend/llgraph/support.py
@@ -156,3 +156,11 @@
         call_args.append(n)
     assert i == len(args)
     return call_args
+
+def addr_add_bytes(addr, ofs):
+    if (isinstance(ofs, int) and
+            getattr(addr.adr.ptr._TYPE.TO, 'OF', None) == lltype.Char):
+        return addr + ofs
+    ptr = rffi.cast(rffi.CCHARP, addr.adr)
+    ptr = lltype.direct_ptradd(ptr, ofs)
+    return cast_to_int(ptr)
diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -1,6 +1,3 @@
-from rpython.jit.backend.ppc.codebuilder import b
-import random
-import sys
 from py.test import raises
 
 from rpython.jit.backend.ppc.form import Form, FormException
@@ -28,18 +25,18 @@
     w = a.insts[-1].assemble()
     return struct.pack('>i', w)
 
-class TestForm(Form):
+class TForm(Form):
     fieldmap = test_fieldmap
 
 class TestForms(object):
     def test_bitclash(self):
-        raises(FormException, TestForm, 'h', 'hh')
-        raises(FormException, TestForm,
+        raises(FormException, TForm, 'h', 'hh')
+        raises(FormException, TForm,
                Field('t1', 0, 0), Field('t2', 0, 0))
 
     def test_basic(self):
         class T(Assembler):
-            i = TestForm('h', 'l')()
+            i = TForm('h', 'l')()
             j = i(h=1)
             k = i(l=3)
             raises(FormException, k, l=0)
@@ -56,7 +53,7 @@
 
     def test_defdesc(self):
         class T(Assembler):
-            i = TestForm('hh', 'hl', 'lh', 'll')()
+            i = TForm('hh', 'hl', 'lh', 'll')()
             i.default(hl=0).default(hh=1)
             insts = []
         a = T()
diff --git a/rpython/jit/backend/ppc/test/test_ppc.py b/rpython/jit/backend/ppc/test/test_ppc.py
--- a/rpython/jit/backend/ppc/test/test_ppc.py
+++ b/rpython/jit/backend/ppc/test/test_ppc.py
@@ -199,9 +199,9 @@
             a.load_imm(r10, call_addr)
         elif IS_BIG_ENDIAN:
             # load the 3-words descriptor
-            a.load_from_addr(r10, call_addr)
-            a.load_from_addr(r2, call_addr+WORD)
-            a.load_from_addr(r11, call_addr+2*WORD)
+            a.load_from_addr(r10, SCRATCH2, call_addr)
+            a.load_from_addr(r2, SCRATCH2, call_addr+WORD)
+            a.load_from_addr(r11, SCRATCH2, call_addr+2*WORD)
         else:
             # no descriptor on little-endian, but the ABI says r12 must
             # contain the function pointer
@@ -304,7 +304,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from_addr(r3, addr)
+        a.load_from_addr(r3, SCRATCH2, addr)
         a.blr()
         f = a.get_assembler_function()
         assert f() == 200
diff --git a/rpython/jit/backend/ppc/test/test_regalloc_3.py b/rpython/jit/backend/ppc/test/test_regalloc_3.py
--- a/rpython/jit/backend/ppc/test/test_regalloc_3.py
+++ b/rpython/jit/backend/ppc/test/test_regalloc_3.py
@@ -55,7 +55,7 @@
     i22 = int_or(i3, i9)
     i41 = int_and(i11, -4)
     i42 = int_or(i41, 1)
-    i23 = int_mod(i12, i42)
+    i23 = int_mul(i12, 0)
     i24 = int_is_true(i6)
     i25 = uint_rshift(i15, 6)
     i26 = int_or(-4, i25)
@@ -63,11 +63,11 @@
     i28 = int_sub(-113, i11)
     i29 = int_neg(i7)
     i30 = int_neg(i24)
-    i31 = int_floordiv(i3, 53)
+    i31 = int_mul(i3, 53)
     i32 = int_mul(i28, i27)
     i43 = int_and(i18, -4)
     i44 = int_or(i43, 1)
-    i33 = int_mod(i26, i44)
+    i33 = int_mul(i26, i44)
     i34 = int_or(i27, i19)
     i35 = uint_lt(i13, 1)
     i45 = int_and(i21, 31)
@@ -84,7 +84,7 @@
     assert cpu.get_int_value(deadframe, 0) == 0
     assert cpu.get_int_value(deadframe, 1) == 0
     assert cpu.get_int_value(deadframe, 2) == 0
-    assert cpu.get_int_value(deadframe, 3) == 0
+    assert cpu.get_int_value(deadframe, 3) == 530
     assert cpu.get_int_value(deadframe, 4) == 1
     assert cpu.get_int_value(deadframe, 5) == -7
     assert cpu.get_int_value(deadframe, 6) == 1
@@ -94,7 +94,7 @@
     assert cpu.get_int_value(deadframe, 10) == 1
     assert cpu.get_int_value(deadframe, 11) == 18
     assert cpu.get_int_value(deadframe, 12) == -1
-    assert cpu.get_int_value(deadframe, 13) == 0
+    assert cpu.get_int_value(deadframe, 13) == 1
 
 def test_bug_1():
     cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], '''
@@ -112,7 +112,7 @@
     i43 = int_and(i14, 31)
     i19 = int_lshift(7, i43)
     i20 = int_neg(i19)
-    i21 = int_mod(i3, 1)
+    i21 = int_and(i3, 0)
     i22 = uint_ge(i15, i1)
     i44 = int_and(i16, 31)
     i23 = int_lshift(i8, i44)
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -38,13 +38,10 @@
     # if v2 == 0 unpacks index 0 of param 2
     # if v2 == 1 unpacks index 1 of param 2
     mask = 0
-    if IS_BIG_ENDIAN:
-        not_implemented("no big endian support (yet)")
-    else:
-        if v1 == 0: mask |= 0b01
-        if v1 == 1: mask |= 0b00
-        if v2 == 0: mask |= 0b10
-        if v2 == 1: mask |= 0b00
+    if v1 == 0: mask |= 0b01
+    if v1 == 1: mask |= 0b00
+    if v2 == 0: mask |= 0b10
+    if v2 == 1: mask |= 0b00
     return mask
 
 
@@ -230,26 +227,19 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            for i in range(2): # at most 2 operations
-                if IS_BIG_ENDIAN:
-                    srcoff = (i * osize)
-                    resoff = (i * nsize)
-                else:
-                    if osize == 4:
-                        srcoff = 8 + (i * osize)
-                        resoff = (i * nsize)
-                    else:
-                        srcoff = (i * osize)
-                        resoff = 8 + (i * nsize)
-                if osize == 8:
-                    self.mc.load(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                else:
-                    self.mc.lwa(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                if nsize == 8:
-                    self.mc.store(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-                else:
-                    self.mc.stw(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-
+            for j in range(2): # at most 2 operations
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // osize) - 1 - i
+                off += osize * i
+                self._load_from_sp(r.SCRATCH.value, osize, off)
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // nsize) - 1 - i
+                off += nsize * i
+                self._store_to_sp(r.SCRATCH.value, nsize, off+16)
             self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
@@ -313,10 +303,7 @@
         acc = accumloc.value
         if arg.type == FLOAT:
             # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            self.mc.xxpermdi(tgt, acc, acc, 0b10)
             if op == '+':
                 self.mc.xsadddp(tgt, tgt, acc)
             elif op == '*':
@@ -471,23 +458,26 @@
         res, l0, off = arglocs
         size = op.bytesize
 
+        idx = 0
+        size = op.bytesize
+        if not IS_BIG_ENDIAN:
+            idx = (16 // size) - 1 - idx
+        idx *= size
+        self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
+        if size == 8:
+            idx = 1
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            idx *= size
+            self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
         self.mc.load_imm(r.SCRATCH2, off.value)
         self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
         if size == 1:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltb(res.value, res.value, 0b0000)
-            else:
-                self.mc.vspltb(res.value, res.value, 0b1111)
+            self.mc.vspltb(res.value, res.value, 0b0000)
         elif size == 2:
-            if IS_BIG_ENDIAN:
-                self.mc.vsplth(res.value, res.value, 0b000)
-            else:
-                self.mc.vsplth(res.value, res.value, 0b111)
+            self.mc.vsplth(res.value, res.value, 0b000)
         elif size == 4:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltw(res.value, res.value, 0b00)
-            else:
-                self.mc.vspltw(res.value, res.value, 0b11)
+            self.mc.vspltw(res.value, res.value, 0b00)
         elif size == 8:
             pass
         else:
@@ -496,7 +486,6 @@
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
-        srcidx = srcidxloc.value
         residx = residxloc.value
         count = countloc.value
         res = resultloc.value
@@ -531,6 +520,37 @@
                 self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
         self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
+    def _load_from_sp(self, res, size, off):
+        if size == 8:
+            self.mc.load(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.lwa(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.lha(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.lbz(res, r.SP.value, off)
+            self.mc.extsb(res, res)
+            return True
+        return False
+
+    def _store_to_sp(self, res, size, off):
+        if size == 8:
+            self.mc.store(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.stw(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.sth(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.stb(res, r.SP.value, off)
+            return True
+        return False
+
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc, idxloc, countloc, sizeloc = arglocs
@@ -539,6 +559,7 @@
         src = srcloc.value
         size = sizeloc.value
         count = countloc.value
+        newsize = op.bytesize
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
@@ -548,18 +569,7 @@
             if not IS_BIG_ENDIAN:
                 idx = (16 // size) - 1 - idx
             off += size * idx
-            if size == 8:
-                self.mc.load(res, r.SP.value, off)
-                return
-            elif size == 4:
-                self.mc.lwa(res, r.SP.value, off)
-                return
-            elif size == 2:
-                self.mc.lha(res, r.SP.value, off)
-                return
-            elif size == 1:
-                self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
+            if self._load_from_sp(res, size, off):
                 return
         else:
             # count is not 1, but only 2 is supported for i32
@@ -571,16 +581,21 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            if count * size == 8:
+            for j in range(count):
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j+idx
                 if not IS_BIG_ENDIAN:
-                    endian_off = 8
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._load_from_sp(r.SCRATCH.value, size, off)
                 off = PARAM_SAVE_AREA_OFFSET
-                off = off + endian_off - (idx * size)
-                assert idx * size + 8 <= 16
-                self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-                return
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._store_to_sp(r.SCRATCH.value, newsize, off+16)
+            self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+            return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
@@ -591,10 +606,8 @@
         vec = vloc.value
         res = resloc.value
         src = srcloc.value
-        count = countloc.value
         residx = residxloc.value
         srcidx = srcidxloc.value
-        size = op.bytesize
         # srcloc is always a floating point register f, this means it is
         # vsr[0] == valueof(f)
         if srcidx == 0:
@@ -864,10 +877,6 @@
             mc.load_imm(l0, arg.value)
         else:
             l0 = self.ensure_reg(arg)
-        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-        size = op.bytesize
-        if size == 8:
-            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
         res = self.force_allocate_vector_reg(op)
         return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1397,5 +1397,6 @@
             assert op not in dups
             dups.add(op)
 
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1190,6 +1190,8 @@
     # Uh, that should be moved to vector_ext really!
     _cast_ops['CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
     _cast_ops['VEC_CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+    _cast_ops['CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
+    _cast_ops['VEC_CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -75,16 +75,6 @@
     request.cls.a
     return rs
 
-
-def rdiv(v1,v2):
-    # TODO unused, interpeting this on top of llgraph does not work correctly
-    try:
-        return v1 / v2
-    except ZeroDivisionError:
-        if v1 == v2 == 0.0:
-            return rfloat.NAN
-        return rfloat.copysign(rfloat.INFINITY, v1 * v2)
-
 class VectorizeTests(object):
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -292,7 +282,7 @@
                 myjitdriver.jit_merge_point()
                 a = va[i]
                 b = vb[i]
-                ec = intmask(a) + intmask(b)
+                ec = intmask(intmask(a) + intmask(b))
                 va[i] = rffi.r_short(ec)
                 i += 1
 
@@ -544,33 +534,6 @@
         res = self.meta_interp(f, [i], vec=True)
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3),(25,2.5,0.3)])
-    def test_list_vectorize(self,i,v1,v2):
-        myjitdriver = JitDriver(greens = [],
-                                reds = 'auto')
-        class ListF(object):
-            def __init__(self, size, init):
-                self.list = [init] * size
-            def __getitem__(self, key):
-                return self.list[key]
-            def __setitem__(self, key, value):
-                self.list[key] = value
-        def f(d, v1, v2):
-            a = ListF(d, v1)
-            b = ListF(d, v2)
-            i = 0
-            while i < d:
-                myjitdriver.jit_merge_point()
-                a[i] = a[i] + b[i]
-                i += 1
-            s = 0
-            for i in range(d):
-                s += a[i]
-            return s
-        res = self.meta_interp(f, [i,v1,v2], vec=True, vec_all=True)
-        # sum helps to generate the rounding error of floating points
-        # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
-        assert res == f(i,v1,v2) == sum([v1+v2]*i)
 
     @py.test.mark.parametrize('size',[12])
     def test_body_multiple_accesses(self, size):
@@ -880,17 +843,18 @@
                 j += 4
                 i += 8
 
-        va = alloc_raw_storage(4*30, zero=True)
-        vb = alloc_raw_storage(8*30, zero=True)
-        for i,v in enumerate([1]*30):
+        count = 32
+        va = alloc_raw_storage(4*count, zero=True)
+        vb = alloc_raw_storage(8*count, zero=True)
+        for i,v in enumerate([1,2,3,4]*(count/4)):
             raw_storage_setitem(va, i*4, rffi.cast(rffi.INT,v))
-        for i,v in enumerate([-9.0]*30):
+        for i,v in enumerate([-1.0,-2.0,-3.0,-4.0]*(count/4)):
             raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
-        vc = alloc_raw_storage(8*30, zero=True)
-        self.meta_interp(f, [8*30, va, vb, vc], vec=True)
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
 
-        for i in range(30):
-            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == -8.0
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
 
         free_raw_storage(va)
         free_raw_storage(vb)
@@ -898,4 +862,14 @@
 
 
 class TestLLtype(LLJitMixin, VectorizeTests):
-    pass
+    # skip some tests on this backend
+    def test_unpack_f(self):
+        pass
+    def test_unpack_i64(self):
+        pass
+    def test_unpack_i(self):
+        pass
+    def test_unpack_several(self):
+        pass
+    def test_vec_int_sum(self):
+        pass
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -323,50 +323,48 @@
 
     def __add__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x + y)
     __radd__ = __add__
 
     def __sub__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x - y)
 
     def __rsub__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # may be a float
         return self._widen(other, x - y)
 
     def __mul__(self, other):
         x = long(self)
-        if not isinstance(other, (int, long)):
-            return x * other
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x * y)
     __rmul__ = __mul__
 
     def __div__(self, other):
         x = long(self)
-        y = long(other)
-        return self._widen(other, x // y)
+        y = other      # may be a float
+        return self._widen(other, x / y)
 
     __floordiv__ = __div__
 
     def __rdiv__(self, other):
         y = long(self)
-        x = long(other)
-        return self._widen(other, x // y)
+        x = other      # may be a float
+        return self._widen(other, x / y)
 
     __rfloordiv__ = __rdiv__
 
     def __mod__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __rmod__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __divmod__(self, other):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -679,21 +679,22 @@
         raise OSError(error, "readdir failed")
     return result
 
-def fdlistdir(dirfd):
-    """
-    Like listdir(), except that the directory is specified as an open
-    file descriptor.
+if not _WIN32:
+    def fdlistdir(dirfd):
+        """
+        Like listdir(), except that the directory is specified as an open
+        file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.  To emulate the
-    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
-    file descriptor.
-    """
-    dirp = c_fdopendir(dirfd)
-    if not dirp:
-        error = get_saved_errno()
-        c_close(dirfd)
-        raise OSError(error, "opendir failed")
-    return _listdir(dirp, rewind=True)
+        Note: fdlistdir() closes the file descriptor.  To emulate the
+        Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+        file descriptor.
+        """
+        dirp = c_fdopendir(dirfd)
+        if not dirp:
+            error = get_saved_errno()
+            c_close(dirfd)
+            raise OSError(error, "opendir failed")
+        return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1166,6 +1167,10 @@
 @replace_os_function('pipe')
 def pipe(flags=0):
     # 'flags' might be ignored.  Check the result.
+    # The handles returned are always inheritable on Posix.
+    # The situation on Windows is not completely clear: I think
+    # it should always return non-inheritable handles, but CPython
+    # uses SECURITY_ATTRIBUTES to ensure that and we don't.
     if _WIN32:
         # 'flags' ignored
         ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
@@ -2146,8 +2151,9 @@
         handle_posix_error('mknodat', error)
 
 
-eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=[r"""
+if not _WIN32:
+    eci_inheritable = eci.merge(ExternalCompilationInfo(
+        separate_module_sources=[r"""
 #include <errno.h>
 #include <sys/ioctl.h>
 
@@ -2200,10 +2206,6 @@
 RPY_EXTERN
 int rpy_dup_noninheritable(int fd)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUPFD_CLOEXEC
     return fcntl(fd, F_DUPFD_CLOEXEC, 0);
 #else
@@ -2221,10 +2223,6 @@
 RPY_EXTERN
 int rpy_dup2_noninheritable(int fd, int fd2)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUP2FD_CLOEXEC
     return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
 
@@ -2249,33 +2247,41 @@
     return 0;
 #endif
 }
-    """ % {'HAVE_DUP3': HAVE_DUP3}],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
-                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
+        """ % {'HAVE_DUP3': HAVE_DUP3}],
+        post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                           'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n'
+                           ]))
 
-c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
-                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                compilation_info=eci_inheritable)
-c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
-                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                 compilation_info=eci_inheritable)
+    _c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    _c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                    rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                    compilation_info=eci_inheritable)
+    c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                     rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                     compilation_info=eci_inheritable)
 
-def set_inheritable(fd, inheritable):
-    result = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', result)
+    def set_inheritable(fd, inheritable):
+        result = _c_set_inheritable(fd, inheritable)
+        handle_posix_error('set_inheritable', result)
 
-def get_inheritable(fd):
-    res = c_get_inheritable(fd)
-    res = handle_posix_error('get_inheritable', res)
-    return res != 0
+    def get_inheritable(fd):
+        res = _c_get_inheritable(fd)
+        res = handle_posix_error('get_inheritable', res)
+        return res != 0
+
+else:
+    # _WIN32
+    from rpython.rlib.rwin32 import set_inheritable, get_inheritable
+    from rpython.rlib.rwin32 import c_dup_noninheritable
+    from rpython.rlib.rwin32 import c_dup2_noninheritable
+
 
 class SetNonInheritableCache(object):
     """Make one prebuilt instance of this for each path that creates
@@ -2315,20 +2321,20 @@
 
 _pipe2_syscall = ENoSysCache()
 
-post_include_bits=['RPY_EXTERN int _cpu_count(void);']
+post_include_bits=['RPY_EXTERN int rpy_cpu_count(void);']
 # cpu count for linux, windows and mac (+ bsds)
 # note that the code is copied from cpython and split up here
 if sys.platform.startswith('linux'):
     cpucount_eci = ExternalCompilationInfo(includes=["unistd.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 return sysconf(_SC_NPROCESSORS_ONLN);
             }
             """], post_include_bits=post_include_bits)
 elif sys.platform == "win32":
     cpucount_eci = ExternalCompilationInfo(includes=["Windows.h"],
             separate_module_sources=["""
-        RPY_EXTERN int _cpu_count(void) {
+        RPY_EXTERN int rpy_cpu_count(void) {
             SYSTEM_INFO sysinfo;
             GetSystemInfo(&sysinfo);
             return sysinfo.dwNumberOfProcessors;
@@ -2337,7 +2343,7 @@
 else:
     cpucount_eci = ExternalCompilationInfo(includes=["sys/types.h", "sys/sysctl.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 int ncpu = 0;
             #if defined(__DragonFly__) || \
                 defined(__OpenBSD__)   || \
@@ -2355,7 +2361,7 @@
             }
             """], post_include_bits=post_include_bits)
 
-_cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real,
+_cpu_count = rffi.llexternal('rpy_cpu_count', [], rffi.INT_real,
                             compilation_info=cpucount_eci)
 
 def cpu_count():
diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -201,6 +201,8 @@
     # default implementation for platforms without a real unsetenv()
     r_putenv(name, '')
 
+REAL_UNSETENV = False
+
 if hasattr(__import__(os.name), 'unsetenv'):
     os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], rffi.INT,
                                   save_err=rffi.RFFI_SAVE_ERRNO)
@@ -222,3 +224,4 @@
     register_external(r_unsetenv, [str0], annmodel.s_None,
                       export_name='ll_os.ll_os_unsetenv',
                       llimpl=unsetenv_llimpl)
+    REAL_UNSETENV = True
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -641,6 +641,7 @@
 
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
+        # XXX 'traverse' is ignored
         win32traits = make_win32_traits(traits)
         with lltype.scoped_alloc(
                 win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data:
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1062,11 +1062,32 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
-def sock_set_inheritable(fd, inheritable):
-    try:
-        rposix.set_inheritable(fd, inheritable)
-    except OSError as e:
-        raise CSocketError(e.errno)
+if _c.WIN32:
+    def sock_set_inheritable(fd, inheritable):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            rwin32.set_handle_inheritable(handle, inheritable)
+        except WindowsError:
+            raise RSocketError("SetHandleInformation failed")   # xxx
+
+    def sock_get_inheritable(fd):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            return rwin32.get_handle_inheritable(handle)
+        except WindowsError:
+            raise RSocketError("GetHandleInformation failed")   # xxx
+else:
+    def sock_set_inheritable(fd, inheritable):
+        try:
+            rposix.set_inheritable(fd, inheritable)
+        except OSError as e:
+            raise CSocketError(e.errno)
+
+    def sock_get_inheritable(fd):
+        try:
+            return rposix.get_inheritable(fd)
+        except OSError as e:
+            raise CSocketError(e.errno)
 
 class SocketError(Exception):
     applevelerrcls = 'error'
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -112,7 +112,7 @@
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
-                       STD_ERROR_HANDLE
+                       STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -473,3 +473,64 @@
     CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
     GetConsoleScreenBufferInfo = winexternal(
         "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
+
+    _GetHandleInformation = winexternal(
+        'GetHandleInformation', [HANDLE, LPDWORD], BOOL)
+    _SetHandleInformation = winexternal(
+        'SetHandleInformation', [HANDLE, DWORD, DWORD], BOOL)
+
+    def set_inheritable(fd, inheritable):
+        handle = get_osfhandle(fd)
+        set_handle_inheritable(handle, inheritable)
+
+    def set_handle_inheritable(handle, inheritable):
+        assert lltype.typeOf(handle) is HANDLE
+        if inheritable:
+            flags = HANDLE_FLAG_INHERIT
+        else:
+            flags = 0
+        if not _SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags):
+            raise lastSavedWindowsError("SetHandleInformation")
+
+    def get_inheritable(fd):
+        handle = get_osfhandle(fd)
+        return get_handle_inheritable(handle)
+
+    def get_handle_inheritable(handle):
+        assert lltype.typeOf(handle) is HANDLE
+        pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
+        try:
+            if not _GetHandleInformation(handle, pflags):
+                raise lastSavedWindowsError("GetHandleInformation")
+            flags = pflags[0]
+        finally:
+            lltype.free(pflags, flavor='raw')
+        return (flags & HANDLE_FLAG_INHERIT) != 0
+
+    _GetFileType = winexternal('GetFileType', [HANDLE], DWORD)
+
+    def c_dup_noninheritable(fd1):
+        from rpython.rlib.rposix import c_dup
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        fd2 = c_dup(fd1)     # the inheritable version
+        if fd2 >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return fd2
+
+    def c_dup2_noninheritable(fd1, fd2):
+        from rpython.rlib.rposix import c_dup2
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        res = c_dup2(fd1, fd2)     # the inheritable version
+        if res >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return res
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -602,3 +602,12 @@
 
         assert r_uint64(self._64_umax) + r_uint64(1) == r_uint64(0)
         assert r_uint64(0) - r_uint64(1) == r_uint64(self._64_umax)
+
+    def test_operation_with_float(self):
+        def f(x):
+            assert r_longlong(x) + 0.5 == 43.5
+            assert r_longlong(x) - 0.5 == 42.5
+            assert r_longlong(x) * 0.5 == 21.5
+            assert r_longlong(x) / 0.8 == 53.75
+        f(43)
+        interpret(f, [43])
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -470,11 +470,18 @@
         assert rposix.is_valid_fd(fd) == 0
 
     def test_putenv(self):
+        from rpython.rlib import rposix_environ
+
         def f():
             rposix.putenv(self.path, self.path)
             rposix.unsetenv(self.path)
 
-        interpret(f, []) # does not crash
+        interpret(f, [],     # does not crash
+                  malloc_check=rposix_environ.REAL_UNSETENV)
+        # If we have a real unsetenv(), check that it frees the string
+        # kept alive by putenv().  Otherwise, we can't check that,
+        # because unsetenv() will keep another string alive itself.
+    test_putenv.dont_track_allocations = True
 
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):
@@ -604,6 +611,9 @@
 def test_SetNonInheritableCache():
     cache = rposix.SetNonInheritableCache()
     fd1, fd2 = os.pipe()
+    if sys.platform == 'win32':
+        rposix.set_inheritable(fd1, True)
+        rposix.set_inheritable(fd2, True)
     assert rposix.get_inheritable(fd1) == True
     assert rposix.get_inheritable(fd1) == True
     assert cache.cached_inheritable == -1
@@ -616,6 +626,24 @@
     os.close(fd1)
     os.close(fd2)
 
+def test_dup_dup2_non_inheritable():
+    for preset in [False, True]:
+        fd1, fd2 = os.pipe()
+        rposix.set_inheritable(fd1, preset)
+        rposix.set_inheritable(fd2, preset)
+        fd3 = rposix.dup(fd1, True)
+        assert rposix.get_inheritable(fd3) == True
+        fd4 = rposix.dup(fd1, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd4, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd3, True)
+        assert rposix.get_inheritable(fd3) == True
+        os.close(fd1)
+        os.close(fd2)
+        os.close(fd3)
+        os.close(fd4)
+
 def test_sync():
     if sys.platform != 'win32':
         rposix.sync()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -124,8 +124,8 @@
         py.test.skip('No socketpair on Windows')
     for inh in [False, True]:
         s1, s2 = socketpair(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
-        assert rposix.get_inheritable(s2.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s2.fd) == inh
         s1.close()
         s2.close()
 
@@ -391,7 +391,7 @@
 def test_inheritable():
     for inh in [False, True]:
         s1 = RSocket(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
         s1.close()
 
 def test_getaddrinfo_http():
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -62,7 +62,7 @@
         guard_dict = {}
         for loop_no, loop in enumerate(loops):
             for op in loop.operations:
-                if op.name.startswith('guard_'):
+                if op.name.startswith('guard_') or op.name.startswith('vec_guard_'):
                     guard_dict[int(op.descr[len('<Guard0x'):-1], 16)] = (op, loop)
         for loop in loops:
             if loop.comment:

From pypy.commits at gmail.com  Wed Nov  9 10:51:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:51:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <582345e8.49151c0a.9962c.9228@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88276:007445aad81a
Date: 2016-11-09 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/007445aad81a/

Log:	merge heads

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1068,9 +1068,6 @@
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
 
-    def wrap_fsdecoded(self, x):
-        return self.fsdecode(self.newbytes(x))
-
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
@@ -1680,6 +1677,9 @@
             w_obj = self.fsdecode(w_obj)
         return self.unicode0_w(w_obj)
 
+    def wrap_fsdecoded(self, x):
+        return self.fsdecode(self.newbytes(x))
+
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
         # For cases where you need to accept bools and ints and nothing
diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -0,0 +1,80 @@
+import sys
+from pypy.interpreter.error import OperationError
+
+class BaseFSEncodeTest:
+
+    def setup_class(cls):
+        space = cls.space
+        cls.testfn_unencodable = get_unencodable()
+        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
+        cls.special_char = get_special_char()
+        cls.w_special_char = space.wrap(cls.special_char)
+
+def get_unencodable():
+    """Copy of the stdlib's support.TESTFN_UNENCODABLE:
+
+    A filename (py3k str type) that should *not* be able to be encoded
+    by the filesystem encoding (in strict mode). It can be None if we
+    cannot generate such filename.
+    """
+    testfn_unencodable = None
+    testfn = u'test_tmp'
+
+    if sys.platform == 'win32':
+        testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
+    elif sys.platform != 'darwin':
+        try:
+            '\xff'.decode(sys.getfilesystemencoding())
+        except UnicodeDecodeError:
+            testfn_unencodable = testfn + u'-\udcff'
+    return testfn_unencodable
+
+def get_special_char():
+    """Copy of the stdlib's test_imp.test_issue5604 special_char:
+
+    A non-ascii filename (py3k str type) that *should* be able to be
+    encoded by the filesystem encoding (in strict mode). It can be None
+    if we cannot generate such filename.
+    """
+    fsenc = sys.getfilesystemencoding()
+    # covers utf-8 and Windows ANSI code pages one non-space symbol from
+    # every page (http://en.wikipedia.org/wiki/Code_page)
+    known_locales = {
+        'utf-8' : b'\xc3\xa4',
+        'cp1250' : b'\x8C',
+        'cp1251' : b'\xc0',
+        'cp1252' : b'\xc0',
+        'cp1253' : b'\xc1',
+        'cp1254' : b'\xc0',
+        'cp1255' : b'\xe0',
+        'cp1256' : b'\xe0',
+        'cp1257' : b'\xc0',
+        'cp1258' : b'\xc0',
+        }
+
+    if sys.platform == 'darwin':
+        # Mac OS X uses the Normal Form D decomposition
+        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
+        special_char = b'a\xcc\x88'
+    else:
+        special_char = known_locales.get(fsenc)
+
+    if special_char:
+        return special_char.decode(fsenc)
+
+class TestFSEncode(BaseFSEncodeTest):
+    def test_fsencode_fsdecode(self):
+        space = self.space
+        strs = [u"/home/bar/baz", u"c:\\"]
+        if self.special_char:
+            strs.append(self.special_char)
+        for st in strs:
+            # check roundtrip
+            w_st = space.newunicode(st)
+            w_enc = space.fsencode(w_st)
+            w_st2 = space.fsdecode(w_enc)
+            assert space.eq_w(w_st, w_st2)
+            assert space.fsdecode_w(w_enc) == st
+
+            assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
+            assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -42,7 +42,7 @@
     w_name = space.wrap(name)
     module = Module(space, w_name)
     if path is not None:
-        module.setdictvalue(space, '__file__', space.wrap(path))
+        module.setdictvalue(space, '__file__', space.wrap_fsdecoded(path))
     module.setdictvalue(space, 'ffi', space.wrap(ffi))
     module.setdictvalue(space, 'lib', space.wrap(lib))
     w_modules_dict = space.sys.get('modules')
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1597,7 +1597,7 @@
             libssl_SSL_CTX_set_default_passwd_cb_userdata(
                 self.ctx, None)
 
-    @unwrap_spec(filepath=str)
+    @unwrap_spec(filepath='fsencode')
     def load_dh_params_w(self, space, filepath):
         bio = libssl_BIO_new_file(filepath, "r")
         if not bio:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -1,6 +1,8 @@
 from rpython.tool.udir import udir
 import os
 
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
+
 class AppTestSSL:
     spaceconfig = dict(usemodules=('_ssl', '_socket', 'select', 'struct',
                                    'binascii', 'thread'))
@@ -341,10 +343,11 @@
             """)
 
 
-class AppTestContext:
+class AppTestContext(BaseFSEncodeTest):
     spaceconfig = dict(usemodules=('_ssl',))
 
     def setup_class(cls):
+        BaseFSEncodeTest.setup_class.im_func(cls)
         tmpfile = udir / "tmpfile.pem"
         tmpfile.write(SSL_CERTIFICATE + SSL_PRIVATE_KEY)
         cls.w_keycert = cls.space.wrap(str(tmpfile))
@@ -366,8 +369,14 @@
         tmpfile = udir / "python.org.pem"
         tmpfile.write(SVN_PYTHON_ORG_ROOT_CERT)
         cls.w_python_org_cert = cls.space.wrap(str(tmpfile))
-        cls.w_dh512 = cls.space.wrap(os.path.join(
-            os.path.dirname(__file__), 'dh512.pem'))
+        tmpfile = udir / cls.special_char
+        fn = os.path.join(
+            os.path.dirname(__file__), 'dh512.pem')
+        with file(fn) as f:
+            s = f.read()
+        tmpfile.write(s)
+        cls.w_dh512 = cls.space.wrap(fn)
+        cls.w_dh512special = cls.space.wrap(str(tmpfile))
 
     def test_load_cert_chain(self):
         import _ssl, errno
@@ -443,6 +452,9 @@
         exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
         assert exc.value.errno == errno.ENOENT
 
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.load_dh_params(self.dh512special)
+
     def test_set_ecdh_curve(self):
         import _ssl
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1511,7 +1511,7 @@
     from pypy.module._cffi_backend import cffi1_module
     cffi1_module.load_cffi1_module(space, name, path, initptr)
 
- at unwrap_spec(path=str, name=str)
+ at unwrap_spec(path='fsencode', name=str)
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
     # modules (cpyext) and to load CFFI-style extension modules
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -47,7 +47,7 @@
     state.package_context = None, None
 
     if f_path is not None:
-        dict_w = {'__file__': space.wrap(f_path)}
+        dict_w = {'__file__': space.wrap_fsdecoded(f_path)}
     else:
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_mod, modname)
diff --git a/pypy/module/imp/test/support.py b/pypy/module/imp/test/support.py
deleted file mode 100644
--- a/pypy/module/imp/test/support.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import sys
-
-class BaseImportTest:
-
-    def setup_class(cls):
-        space = cls.space
-        cls.testfn_unencodable = get_unencodable()
-        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
-        cls.special_char = get_special_char()
-        cls.w_special_char = space.wrap(cls.special_char)
-
-def get_unencodable():
-    """Copy of the stdlib's support.TESTFN_UNENCODABLE:
-
-    A filename (py3k str type) that should *not* be able to be encoded
-    by the filesystem encoding (in strict mode). It can be None if we
-    cannot generate such filename.
-    """
-    testfn_unencodable = None
-    testfn = u'test_tmp'
-
-    if sys.platform == 'win32':
-        testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
-    elif sys.platform != 'darwin':
-        try:
-            '\xff'.decode(sys.getfilesystemencoding())
-        except UnicodeDecodeError:
-            testfn_unencodable = testfn + u'-\udcff'
-    return testfn_unencodable
-
-def get_special_char():
-    """Copy of the stdlib's test_imp.test_issue5604 special_char:
-
-    A non-ascii filename (py3k str type) that *should* be able to be
-    encoded by the filesystem encoding (in strict mode). It can be None
-    if we cannot generate such filename.
-    """
-    fsenc = sys.getfilesystemencoding()
-    # covers utf-8 and Windows ANSI code pages one non-space symbol from
-    # every page (http://en.wikipedia.org/wiki/Code_page)
-    known_locales = {
-        'utf-8' : b'\xc3\xa4',
-        'cp1250' : b'\x8C',
-        'cp1251' : b'\xc0',
-        'cp1252' : b'\xc0',
-        'cp1253' : b'\xc1',
-        'cp1254' : b'\xc0',
-        'cp1255' : b'\xe0',
-        'cp1256' : b'\xe0',
-        'cp1257' : b'\xc0',
-        'cp1258' : b'\xc0',
-        }
-
-    if sys.platform == 'darwin':
-        # Mac OS X uses the Normal Form D decomposition
-        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
-        special_char = b'a\xcc\x88'
-    else:
-        special_char = known_locales.get(fsenc)
-
-    if special_char:
-        return special_char.decode(fsenc)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -4,7 +4,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pycode import PyCode
-from pypy.module.imp.test.support import BaseImportTest
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
 from rpython.tool.udir import udir
 from rpython.rlib import streamio
 from pypy.tool.option import make_config
@@ -177,13 +177,13 @@
     """)
 
 
-class AppTestImport(BaseImportTest):
+class AppTestImport(BaseFSEncodeTest):
     spaceconfig = {
         "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
     }
 
     def setup_class(cls):
-        BaseImportTest.setup_class.im_func(cls)
+        BaseFSEncodeTest.setup_class.im_func(cls)
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.w_saved_modules = _setup(cls)
         #XXX Compile class

From pypy.commits at gmail.com  Wed Nov  9 10:51:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 07:51:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582345e4.0a4cc20a.3a027.12fd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88274:43cb58034ee2
Date: 2016-11-09 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/43cb58034ee2/

Log:	hg merge default

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('PyMODINIT_FUNC')
+        prnt('void')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:

From pypy.commits at gmail.com  Wed Nov  9 11:02:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 08:02:59 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: open new release branch for
 pypy3.5-5.x
Message-ID: <582348b3.88711c0a.1ea76.0c80@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r88277:b35e13efd43f
Date: 2016-11-09 17:02 +0100
http://bitbucket.org/pypy/pypy/changeset/b35e13efd43f/

Log:	open new release branch for pypy3.5-5.x


From pypy.commits at gmail.com  Wed Nov  9 11:04:58 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 08:04:58 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge current head (py3.5)
 into release branch
Message-ID: <5823492a.c1341c0a.942da.5a7f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r88278:33d015479c3c
Date: 2016-11-09 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/33d015479c3c/

Log:	merge current head (py3.5) into release branch

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('PyMODINIT_FUNC')
+        prnt('void')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -34,7 +34,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -110,18 +115,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('builtins'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -129,6 +139,9 @@
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1068,9 +1068,6 @@
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
 
-    def wrap_fsdecoded(self, x):
-        return self.fsdecode(self.newbytes(x))
-
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
@@ -1680,6 +1677,9 @@
             w_obj = self.fsdecode(w_obj)
         return self.unicode0_w(w_obj)
 
+    def wrap_fsdecoded(self, x):
+        return self.fsdecode(self.newbytes(x))
+
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
         # For cases where you need to accept bools and ints and nothing
diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -0,0 +1,80 @@
+import sys
+from pypy.interpreter.error import OperationError
+
+class BaseFSEncodeTest:
+
+    def setup_class(cls):
+        space = cls.space
+        cls.testfn_unencodable = get_unencodable()
+        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
+        cls.special_char = get_special_char()
+        cls.w_special_char = space.wrap(cls.special_char)
+
+def get_unencodable():
+    """Copy of the stdlib's support.TESTFN_UNENCODABLE:
+
+    A filename (py3k str type) that should *not* be able to be encoded
+    by the filesystem encoding (in strict mode). It can be None if we
+    cannot generate such filename.
+    """
+    testfn_unencodable = None
+    testfn = u'test_tmp'
+
+    if sys.platform == 'win32':
+        testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
+    elif sys.platform != 'darwin':
+        try:
+            '\xff'.decode(sys.getfilesystemencoding())
+        except UnicodeDecodeError:
+            testfn_unencodable = testfn + u'-\udcff'
+    return testfn_unencodable
+
+def get_special_char():
+    """Copy of the stdlib's test_imp.test_issue5604 special_char:
+
+    A non-ascii filename (py3k str type) that *should* be able to be
+    encoded by the filesystem encoding (in strict mode). It can be None
+    if we cannot generate such filename.
+    """
+    fsenc = sys.getfilesystemencoding()
+    # covers utf-8 and Windows ANSI code pages one non-space symbol from
+    # every page (http://en.wikipedia.org/wiki/Code_page)
+    known_locales = {
+        'utf-8' : b'\xc3\xa4',
+        'cp1250' : b'\x8C',
+        'cp1251' : b'\xc0',
+        'cp1252' : b'\xc0',
+        'cp1253' : b'\xc1',
+        'cp1254' : b'\xc0',
+        'cp1255' : b'\xe0',
+        'cp1256' : b'\xe0',
+        'cp1257' : b'\xc0',
+        'cp1258' : b'\xc0',
+        }
+
+    if sys.platform == 'darwin':
+        # Mac OS X uses the Normal Form D decomposition
+        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
+        special_char = b'a\xcc\x88'
+    else:
+        special_char = known_locales.get(fsenc)
+
+    if special_char:
+        return special_char.decode(fsenc)
+
+class TestFSEncode(BaseFSEncodeTest):
+    def test_fsencode_fsdecode(self):
+        space = self.space
+        strs = [u"/home/bar/baz", u"c:\\"]
+        if self.special_char:
+            strs.append(self.special_char)
+        for st in strs:
+            # check roundtrip
+            w_st = space.newunicode(st)
+            w_enc = space.fsencode(w_st)
+            w_st2 = space.fsdecode(w_enc)
+            assert space.eq_w(w_st, w_st2)
+            assert space.fsdecode_w(w_enc) == st
+
+            assert space.fsencode_w(w_enc) == space.bytes_w(w_enc)
+            assert space.eq_w(space.wrap_fsdecoded(space.bytes_w(w_enc)), w_st2)
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -20,7 +20,7 @@
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    lls = rffi.str2charp("sys")
+    lls = rffi.str2charp("sys   # should give a NameError")
     execute_source(lls)
     lltype.free(lls, flavor='raw')
     # did not crash - the same globals
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,6 @@
     def startup(self, space):
         from pypy.module._cffi_backend import embedding
         embedding.glob.space = space
-        embedding.glob.patched_sys = False
 
 
 def get_dict_rtld_constants():
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -42,7 +42,7 @@
     w_name = space.wrap(name)
     module = Module(space, w_name)
     if path is not None:
-        module.setdictvalue(space, '__file__', space.wrap(path))
+        module.setdictvalue(space, '__file__', space.wrap_fsdecoded(path))
     module.setdictvalue(space, 'ffi', space.wrap(ffi))
     module.setdictvalue(space, 'lib', space.wrap(lib))
     w_modules_dict = space.sys.get('modules')
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -45,25 +45,6 @@
     pass
 glob = Global()
 
-def patch_sys(space):
-    # Annoying: CPython would just use the C-level std{in,out,err} as
-    # configured by the main application, for example in binary mode
-    # on Windows or with buffering turned off.  We can't easily do the
-    # same.  Instead, go for the safest bet (but possibly bad for
-    # performance) and open sys.std{in,out,err} unbuffered.  On
-    # Windows I guess binary mode is a better default choice.
-    #
-    # XXX if needed, we could add support for a flag passed to
-    # pypy_init_embedded_cffi_module().
-    if not glob.patched_sys:
-        space.appexec([], """():
-            import os, sys
-            sys.stdin  = sys.__stdin__  = os.fdopen(0, 'rb', 0)
-            sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0)
-            sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0)
-        """)
-        glob.patched_sys = True
-
 
 def pypy_init_embedded_cffi_module(version, init_struct):
     # called from __init__.py
@@ -76,7 +57,6 @@
         must_leave = False
         try:
             must_leave = space.threadlocals.try_enter_thread(space)
-            patch_sys(space)
             load_embedded_cffi_module(space, version, init_struct)
             res = 0
         except OperationError as operr:
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1597,7 +1597,7 @@
             libssl_SSL_CTX_set_default_passwd_cb_userdata(
                 self.ctx, None)
 
-    @unwrap_spec(filepath=str)
+    @unwrap_spec(filepath='fsencode')
     def load_dh_params_w(self, space, filepath):
         bio = libssl_BIO_new_file(filepath, "r")
         if not bio:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -1,6 +1,8 @@
 from rpython.tool.udir import udir
 import os
 
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
+
 class AppTestSSL:
     spaceconfig = dict(usemodules=('_ssl', '_socket', 'select', 'struct',
                                    'binascii', 'thread'))
@@ -341,10 +343,11 @@
             """)
 
 
-class AppTestContext:
+class AppTestContext(BaseFSEncodeTest):
     spaceconfig = dict(usemodules=('_ssl',))
 
     def setup_class(cls):
+        BaseFSEncodeTest.setup_class.im_func(cls)
         tmpfile = udir / "tmpfile.pem"
         tmpfile.write(SSL_CERTIFICATE + SSL_PRIVATE_KEY)
         cls.w_keycert = cls.space.wrap(str(tmpfile))
@@ -366,8 +369,14 @@
         tmpfile = udir / "python.org.pem"
         tmpfile.write(SVN_PYTHON_ORG_ROOT_CERT)
         cls.w_python_org_cert = cls.space.wrap(str(tmpfile))
-        cls.w_dh512 = cls.space.wrap(os.path.join(
-            os.path.dirname(__file__), 'dh512.pem'))
+        tmpfile = udir / cls.special_char
+        fn = os.path.join(
+            os.path.dirname(__file__), 'dh512.pem')
+        with file(fn) as f:
+            s = f.read()
+        tmpfile.write(s)
+        cls.w_dh512 = cls.space.wrap(fn)
+        cls.w_dh512special = cls.space.wrap(str(tmpfile))
 
     def test_load_cert_chain(self):
         import _ssl, errno
@@ -443,6 +452,9 @@
         exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
         assert exc.value.errno == errno.ENOENT
 
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
+        ctx.load_dh_params(self.dh512special)
+
     def test_set_ecdh_curve(self):
         import _ssl
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLS)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1511,7 +1511,7 @@
     from pypy.module._cffi_backend import cffi1_module
     cffi1_module.load_cffi1_module(space, name, path, initptr)
 
- at unwrap_spec(path=str, name=str)
+ at unwrap_spec(path='fsencode', name=str)
 def load_extension_module(space, path, name):
     # note: this is used both to load CPython-API-style C extension
     # modules (cpyext) and to load CFFI-style extension modules
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -47,7 +47,7 @@
     state.package_context = None, None
 
     if f_path is not None:
-        dict_w = {'__file__': space.wrap(f_path)}
+        dict_w = {'__file__': space.wrap_fsdecoded(f_path)}
     else:
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_mod, modname)
diff --git a/pypy/module/imp/test/support.py b/pypy/module/imp/test/support.py
deleted file mode 100644
--- a/pypy/module/imp/test/support.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import sys
-
-class BaseImportTest:
-
-    def setup_class(cls):
-        space = cls.space
-        cls.testfn_unencodable = get_unencodable()
-        cls.w_testfn_unencodable = space.wrap(cls.testfn_unencodable)
-        cls.special_char = get_special_char()
-        cls.w_special_char = space.wrap(cls.special_char)
-
-def get_unencodable():
-    """Copy of the stdlib's support.TESTFN_UNENCODABLE:
-
-    A filename (py3k str type) that should *not* be able to be encoded
-    by the filesystem encoding (in strict mode). It can be None if we
-    cannot generate such filename.
-    """
-    testfn_unencodable = None
-    testfn = u'test_tmp'
-
-    if sys.platform == 'win32':
-        testfn_unencodable = testfn + u"-\u5171\u0141\u2661\u0363\uDC80"
-    elif sys.platform != 'darwin':
-        try:
-            '\xff'.decode(sys.getfilesystemencoding())
-        except UnicodeDecodeError:
-            testfn_unencodable = testfn + u'-\udcff'
-    return testfn_unencodable
-
-def get_special_char():
-    """Copy of the stdlib's test_imp.test_issue5604 special_char:
-
-    A non-ascii filename (py3k str type) that *should* be able to be
-    encoded by the filesystem encoding (in strict mode). It can be None
-    if we cannot generate such filename.
-    """
-    fsenc = sys.getfilesystemencoding()
-    # covers utf-8 and Windows ANSI code pages one non-space symbol from
-    # every page (http://en.wikipedia.org/wiki/Code_page)
-    known_locales = {
-        'utf-8' : b'\xc3\xa4',
-        'cp1250' : b'\x8C',
-        'cp1251' : b'\xc0',
-        'cp1252' : b'\xc0',
-        'cp1253' : b'\xc1',
-        'cp1254' : b'\xc0',
-        'cp1255' : b'\xe0',
-        'cp1256' : b'\xe0',
-        'cp1257' : b'\xc0',
-        'cp1258' : b'\xc0',
-        }
-
-    if sys.platform == 'darwin':
-        # Mac OS X uses the Normal Form D decomposition
-        # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
-        special_char = b'a\xcc\x88'
-    else:
-        special_char = known_locales.get(fsenc)
-
-    if special_char:
-        return special_char.decode(fsenc)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -4,7 +4,7 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pycode import PyCode
-from pypy.module.imp.test.support import BaseImportTest
+from pypy.interpreter.test.test_fsencode import BaseFSEncodeTest
 from rpython.tool.udir import udir
 from rpython.rlib import streamio
 from pypy.tool.option import make_config
@@ -177,13 +177,13 @@
     """)
 
 
-class AppTestImport(BaseImportTest):
+class AppTestImport(BaseFSEncodeTest):
     spaceconfig = {
         "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
     }
 
     def setup_class(cls):
-        BaseImportTest.setup_class.im_func(cls)
+        BaseFSEncodeTest.setup_class.im_func(cls)
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.w_saved_modules = _setup(cls)
         #XXX Compile class
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -18,7 +18,7 @@
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
-from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
 from rpython.rlib.objectmodel import Symbolic
 
 class LLAsmInfo(object):
@@ -840,6 +840,11 @@
     vector_arith_code = """
     def bh_vec_{0}_{1}(self, vx, vy, count):
         assert len(vx) == len(vy) == count
+        return [intmask(_vx {2} _vy) for _vx,_vy in zip(vx,vy)]
+    """
+    vector_float_arith_code = """
+    def bh_vec_{0}_{1}(self, vx, vy, count):
+        assert len(vx) == len(vy) == count
         return [_vx {2} _vy for _vx,_vy in zip(vx,vy)]
     """
     exec py.code.Source(vector_arith_code.format('int','add','+')).compile()
@@ -847,11 +852,12 @@
     exec py.code.Source(vector_arith_code.format('int','mul','*')).compile()
     exec py.code.Source(vector_arith_code.format('int','and','&')).compile()
     exec py.code.Source(vector_arith_code.format('int','or','|')).compile()
-    exec py.code.Source(vector_arith_code.format('float','add','+')).compile()
-    exec py.code.Source(vector_arith_code.format('float','sub','-')).compile()
-    exec py.code.Source(vector_arith_code.format('float','mul','*')).compile()
-    exec py.code.Source(vector_arith_code.format('float','truediv','/')).compile()
-    exec py.code.Source(vector_arith_code.format('float','eq','==')).compile()
+
+    exec py.code.Source(vector_float_arith_code.format('float','add','+')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','sub','-')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','mul','*')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','truediv','/')).compile()
+    exec py.code.Source(vector_float_arith_code.format('float','eq','==')).compile()
 
     def bh_vec_float_neg(self, vx, count):
         return [e * -1 for e in vx]
@@ -891,9 +897,6 @@
         return [longlong.getfloatstorage(float(longlong.int2singlefloat(v)))
                 for v in vx]
 
-        a = float(a)
-        return longlong.getfloatstorage(a)
-
     def bh_vec_cast_float_to_int(self, vx, count):
         return [int(x) for x in vx]
 
@@ -934,27 +937,26 @@
         return [heaptracker.int_signext(_vx, ext) for _vx in vx]
 
     def build_load(func):
-        def method(self, struct, offset, scale, disp, descr, _count):
+        def load(self, struct, offset, scale, disp, descr, _count):
             values = []
             count = self.vector_ext.vec_size() // descr.get_item_size_in_bytes()
             assert _count == count
             assert count > 0
-            adr = struct + (offset * scale + disp)
+            adr = support.addr_add_bytes(struct, (offset * scale + disp))
             a = support.cast_arg(lltype.Ptr(descr.A), adr)
             array = a._obj
             for i in range(count):
                 val = support.cast_result(descr.A.OF, array.getitem(i))
                 values.append(val)
             return values
-        return method
+        return load
 
     bh_vec_load_i = build_load(bh_getarrayitem_raw)
     bh_vec_load_f = build_load(bh_getarrayitem_raw)
     del build_load
 
     def bh_vec_store(self, struct, offset, newvalues, scale, disp, descr, count):
-        stride = descr.get_item_size_in_bytes()
-        adr = struct + (offset * scale + disp)
+        adr = support.addr_add_bytes(struct, offset * scale + disp)
         a = support.cast_arg(lltype.Ptr(descr.A), adr)
         array = a._obj
         for i,n in enumerate(newvalues):
@@ -1557,7 +1559,7 @@
             if opname.startswith('vec_'):
                 # pre vector op
                 count = self.current_op.count
-                assert count >= 1
+                assert count >= 0
                 new_args = new_args + (count,)
             result = getattr(self.cpu, 'bh_' + opname)(*new_args)
             if isinstance(result, list):
diff --git a/rpython/jit/backend/llgraph/support.py b/rpython/jit/backend/llgraph/support.py
--- a/rpython/jit/backend/llgraph/support.py
+++ b/rpython/jit/backend/llgraph/support.py
@@ -156,3 +156,11 @@
         call_args.append(n)
     assert i == len(args)
     return call_args
+
+def addr_add_bytes(addr, ofs):
+    if (isinstance(ofs, int) and
+            getattr(addr.adr.ptr._TYPE.TO, 'OF', None) == lltype.Char):
+        return addr + ofs
+    ptr = rffi.cast(rffi.CCHARP, addr.adr)
+    ptr = lltype.direct_ptradd(ptr, ofs)
+    return cast_to_int(ptr)
diff --git a/rpython/jit/backend/ppc/test/test_form.py b/rpython/jit/backend/ppc/test/test_form.py
--- a/rpython/jit/backend/ppc/test/test_form.py
+++ b/rpython/jit/backend/ppc/test/test_form.py
@@ -1,6 +1,3 @@
-from rpython.jit.backend.ppc.codebuilder import b
-import random
-import sys
 from py.test import raises
 
 from rpython.jit.backend.ppc.form import Form, FormException
@@ -28,18 +25,18 @@
     w = a.insts[-1].assemble()
     return struct.pack('>i', w)
 
-class TestForm(Form):
+class TForm(Form):
     fieldmap = test_fieldmap
 
 class TestForms(object):
     def test_bitclash(self):
-        raises(FormException, TestForm, 'h', 'hh')
-        raises(FormException, TestForm,
+        raises(FormException, TForm, 'h', 'hh')
+        raises(FormException, TForm,
                Field('t1', 0, 0), Field('t2', 0, 0))
 
     def test_basic(self):
         class T(Assembler):
-            i = TestForm('h', 'l')()
+            i = TForm('h', 'l')()
             j = i(h=1)
             k = i(l=3)
             raises(FormException, k, l=0)
@@ -56,7 +53,7 @@
 
     def test_defdesc(self):
         class T(Assembler):
-            i = TestForm('hh', 'hl', 'lh', 'll')()
+            i = TForm('hh', 'hl', 'lh', 'll')()
             i.default(hl=0).default(hh=1)
             insts = []
         a = T()
diff --git a/rpython/jit/backend/ppc/test/test_ppc.py b/rpython/jit/backend/ppc/test/test_ppc.py
--- a/rpython/jit/backend/ppc/test/test_ppc.py
+++ b/rpython/jit/backend/ppc/test/test_ppc.py
@@ -199,9 +199,9 @@
             a.load_imm(r10, call_addr)
         elif IS_BIG_ENDIAN:
             # load the 3-words descriptor
-            a.load_from_addr(r10, call_addr)
-            a.load_from_addr(r2, call_addr+WORD)
-            a.load_from_addr(r11, call_addr+2*WORD)
+            a.load_from_addr(r10, SCRATCH2, call_addr)
+            a.load_from_addr(r2, SCRATCH2, call_addr+WORD)
+            a.load_from_addr(r11, SCRATCH2, call_addr+2*WORD)
         else:
             # no descriptor on little-endian, but the ABI says r12 must
             # contain the function pointer
@@ -304,7 +304,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from_addr(r3, addr)
+        a.load_from_addr(r3, SCRATCH2, addr)
         a.blr()
         f = a.get_assembler_function()
         assert f() == 200
diff --git a/rpython/jit/backend/ppc/test/test_regalloc_3.py b/rpython/jit/backend/ppc/test/test_regalloc_3.py
--- a/rpython/jit/backend/ppc/test/test_regalloc_3.py
+++ b/rpython/jit/backend/ppc/test/test_regalloc_3.py
@@ -55,7 +55,7 @@
     i22 = int_or(i3, i9)
     i41 = int_and(i11, -4)
     i42 = int_or(i41, 1)
-    i23 = int_mod(i12, i42)
+    i23 = int_mul(i12, 0)
     i24 = int_is_true(i6)
     i25 = uint_rshift(i15, 6)
     i26 = int_or(-4, i25)
@@ -63,11 +63,11 @@
     i28 = int_sub(-113, i11)
     i29 = int_neg(i7)
     i30 = int_neg(i24)
-    i31 = int_floordiv(i3, 53)
+    i31 = int_mul(i3, 53)
     i32 = int_mul(i28, i27)
     i43 = int_and(i18, -4)
     i44 = int_or(i43, 1)
-    i33 = int_mod(i26, i44)
+    i33 = int_mul(i26, i44)
     i34 = int_or(i27, i19)
     i35 = uint_lt(i13, 1)
     i45 = int_and(i21, 31)
@@ -84,7 +84,7 @@
     assert cpu.get_int_value(deadframe, 0) == 0
     assert cpu.get_int_value(deadframe, 1) == 0
     assert cpu.get_int_value(deadframe, 2) == 0
-    assert cpu.get_int_value(deadframe, 3) == 0
+    assert cpu.get_int_value(deadframe, 3) == 530
     assert cpu.get_int_value(deadframe, 4) == 1
     assert cpu.get_int_value(deadframe, 5) == -7
     assert cpu.get_int_value(deadframe, 6) == 1
@@ -94,7 +94,7 @@
     assert cpu.get_int_value(deadframe, 10) == 1
     assert cpu.get_int_value(deadframe, 11) == 18
     assert cpu.get_int_value(deadframe, 12) == -1
-    assert cpu.get_int_value(deadframe, 13) == 0
+    assert cpu.get_int_value(deadframe, 13) == 1
 
 def test_bug_1():
     cpu, deadframe = run([17, -20, -6, 6, 1, 13, 13, 9, 49, 8], '''
@@ -112,7 +112,7 @@
     i43 = int_and(i14, 31)
     i19 = int_lshift(7, i43)
     i20 = int_neg(i19)
-    i21 = int_mod(i3, 1)
+    i21 = int_and(i3, 0)
     i22 = uint_ge(i15, i1)
     i44 = int_and(i16, 31)
     i23 = int_lshift(i8, i44)
diff --git a/rpython/jit/backend/ppc/vector_ext.py b/rpython/jit/backend/ppc/vector_ext.py
--- a/rpython/jit/backend/ppc/vector_ext.py
+++ b/rpython/jit/backend/ppc/vector_ext.py
@@ -38,13 +38,10 @@
     # if v2 == 0 unpacks index 0 of param 2
     # if v2 == 1 unpacks index 1 of param 2
     mask = 0
-    if IS_BIG_ENDIAN:
-        not_implemented("no big endian support (yet)")
-    else:
-        if v1 == 0: mask |= 0b01
-        if v1 == 1: mask |= 0b00
-        if v2 == 0: mask |= 0b10
-        if v2 == 1: mask |= 0b00
+    if v1 == 0: mask |= 0b01
+    if v1 == 1: mask |= 0b00
+    if v2 == 0: mask |= 0b10
+    if v2 == 1: mask |= 0b00
     return mask
 
 
@@ -230,26 +227,19 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            for i in range(2): # at most 2 operations
-                if IS_BIG_ENDIAN:
-                    srcoff = (i * osize)
-                    resoff = (i * nsize)
-                else:
-                    if osize == 4:
-                        srcoff = 8 + (i * osize)
-                        resoff = (i * nsize)
-                    else:
-                        srcoff = (i * osize)
-                        resoff = 8 + (i * nsize)
-                if osize == 8:
-                    self.mc.load(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                else:
-                    self.mc.lwa(r.SCRATCH.value, r.SP.value, srcoff + PARAM_SAVE_AREA_OFFSET)
-                if nsize == 8:
-                    self.mc.store(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-                else:
-                    self.mc.stw(r.SCRATCH.value, r.SP.value, resoff + PARAM_SAVE_AREA_OFFSET+16)
-
+            for j in range(2): # at most 2 operations
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // osize) - 1 - i
+                off += osize * i
+                self._load_from_sp(r.SCRATCH.value, osize, off)
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // nsize) - 1 - i
+                off += nsize * i
+                self._store_to_sp(r.SCRATCH.value, nsize, off+16)
             self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
@@ -313,10 +303,7 @@
         acc = accumloc.value
         if arg.type == FLOAT:
             # r = (r[0]+r[1],r[0]+r[1])
-            if IS_BIG_ENDIAN:
-                self.mc.xxpermdi(tgt, acc, acc, 0b00)
-            else:
-                self.mc.xxpermdi(tgt, acc, acc, 0b10)
+            self.mc.xxpermdi(tgt, acc, acc, 0b10)
             if op == '+':
                 self.mc.xsadddp(tgt, tgt, acc)
             elif op == '*':
@@ -471,23 +458,26 @@
         res, l0, off = arglocs
         size = op.bytesize
 
+        idx = 0
+        size = op.bytesize
+        if not IS_BIG_ENDIAN:
+            idx = (16 // size) - 1 - idx
+        idx *= size
+        self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
+        if size == 8:
+            idx = 1
+            if not IS_BIG_ENDIAN:
+                idx = (16 // size) - 1 - idx
+            idx *= size
+            self._store_to_sp(l0.value, size, idx+PARAM_SAVE_AREA_OFFSET)
         self.mc.load_imm(r.SCRATCH2, off.value)
         self.mc.lvx(res.value, r.SCRATCH2.value, r.SP.value)
         if size == 1:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltb(res.value, res.value, 0b0000)
-            else:
-                self.mc.vspltb(res.value, res.value, 0b1111)
+            self.mc.vspltb(res.value, res.value, 0b0000)
         elif size == 2:
-            if IS_BIG_ENDIAN:
-                self.mc.vsplth(res.value, res.value, 0b000)
-            else:
-                self.mc.vsplth(res.value, res.value, 0b111)
+            self.mc.vsplth(res.value, res.value, 0b000)
         elif size == 4:
-            if IS_BIG_ENDIAN:
-                self.mc.vspltw(res.value, res.value, 0b00)
-            else:
-                self.mc.vspltw(res.value, res.value, 0b11)
+            self.mc.vspltw(res.value, res.value, 0b00)
         elif size == 8:
             pass
         else:
@@ -496,7 +486,6 @@
     def emit_vec_pack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resultloc, vloc, sourceloc, residxloc, srcidxloc, countloc = arglocs
-        srcidx = srcidxloc.value
         residx = residxloc.value
         count = countloc.value
         res = resultloc.value
@@ -531,6 +520,37 @@
                 self.mc.stb(src, r.SP.value, PARAM_SAVE_AREA_OFFSET+idx)
         self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
 
+    def _load_from_sp(self, res, size, off):
+        if size == 8:
+            self.mc.load(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.lwa(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.lha(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.lbz(res, r.SP.value, off)
+            self.mc.extsb(res, res)
+            return True
+        return False
+
+    def _store_to_sp(self, res, size, off):
+        if size == 8:
+            self.mc.store(res, r.SP.value, off)
+            return True
+        elif size == 4:
+            self.mc.stw(res, r.SP.value, off)
+            return True
+        elif size == 2:
+            self.mc.sth(res, r.SP.value, off)
+            return True
+        elif size == 1:
+            self.mc.stb(res, r.SP.value, off)
+            return True
+        return False
+
     def emit_vec_unpack_i(self, op, arglocs, regalloc):
         assert isinstance(op, VectorOp)
         resloc, srcloc, idxloc, countloc, sizeloc = arglocs
@@ -539,6 +559,7 @@
         src = srcloc.value
         size = sizeloc.value
         count = countloc.value
+        newsize = op.bytesize
         if count == 1:
             assert srcloc.is_vector_reg()
             assert not resloc.is_vector_reg()
@@ -548,18 +569,7 @@
             if not IS_BIG_ENDIAN:
                 idx = (16 // size) - 1 - idx
             off += size * idx
-            if size == 8:
-                self.mc.load(res, r.SP.value, off)
-                return
-            elif size == 4:
-                self.mc.lwa(res, r.SP.value, off)
-                return
-            elif size == 2:
-                self.mc.lha(res, r.SP.value, off)
-                return
-            elif size == 1:
-                self.mc.lbz(res, r.SP.value, off)
-                self.mc.extsb(res, res)
+            if self._load_from_sp(res, size, off):
                 return
         else:
             # count is not 1, but only 2 is supported for i32
@@ -571,16 +581,21 @@
             self.mc.stvx(src, r.SCRATCH2.value, r.SP.value)
             self.mc.load_imm(r.SCRATCH2, PARAM_SAVE_AREA_OFFSET+16)
             self.mc.stvx(res, r.SCRATCH2.value, r.SP.value)
-            if count * size == 8:
+            for j in range(count):
+                off = PARAM_SAVE_AREA_OFFSET
+                i = j+idx
                 if not IS_BIG_ENDIAN:
-                    endian_off = 8
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._load_from_sp(r.SCRATCH.value, size, off)
                 off = PARAM_SAVE_AREA_OFFSET
-                off = off + endian_off - (idx * size)
-                assert idx * size + 8 <= 16
-                self.mc.load(r.SCRATCH.value, r.SP.value, off)
-                self.mc.store(r.SCRATCH.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+16+endian_off)
-                self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
-                return
+                i = j
+                if not IS_BIG_ENDIAN:
+                    i = (16 // size) - 1 - i
+                off += size * i
+                self._store_to_sp(r.SCRATCH.value, newsize, off+16)
+            self.mc.lvx(res, r.SCRATCH2.value, r.SP.value)
+            return
 
         not_implemented("%d bit integer, count %d" % \
                        (size*8, count))
@@ -591,10 +606,8 @@
         vec = vloc.value
         res = resloc.value
         src = srcloc.value
-        count = countloc.value
         residx = residxloc.value
         srcidx = srcidxloc.value
-        size = op.bytesize
         # srcloc is always a floating point register f, this means it is
         # vsr[0] == valueof(f)
         if srcidx == 0:
@@ -864,10 +877,6 @@
             mc.load_imm(l0, arg.value)
         else:
             l0 = self.ensure_reg(arg)
-        mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET)
-        size = op.bytesize
-        if size == 8:
-            mc.store(l0.value, r.SP.value, PARAM_SAVE_AREA_OFFSET+8)
         res = self.force_allocate_vector_reg(op)
         return [res, l0, imm(PARAM_SAVE_AREA_OFFSET)]
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vecopt.py
@@ -1397,5 +1397,6 @@
             assert op not in dups
             dups.add(op)
 
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1190,6 +1190,8 @@
     # Uh, that should be moved to vector_ext really!
     _cast_ops['CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
     _cast_ops['VEC_CAST_FLOAT_TO_INT'] = ('f', 8, 'i', 8, 2)
+    _cast_ops['CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
+    _cast_ops['VEC_CAST_INT_TO_FLOAT'] = ('i', 8, 'f', 8, 2)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -75,16 +75,6 @@
     request.cls.a
     return rs
 
-
-def rdiv(v1,v2):
-    # TODO unused, interpeting this on top of llgraph does not work correctly
-    try:
-        return v1 / v2
-    except ZeroDivisionError:
-        if v1 == v2 == 0.0:
-            return rfloat.NAN
-        return rfloat.copysign(rfloat.INFINITY, v1 * v2)
-
 class VectorizeTests(object):
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -292,7 +282,7 @@
                 myjitdriver.jit_merge_point()
                 a = va[i]
                 b = vb[i]
-                ec = intmask(a) + intmask(b)
+                ec = intmask(intmask(a) + intmask(b))
                 va[i] = rffi.r_short(ec)
                 i += 1
 
@@ -544,33 +534,6 @@
         res = self.meta_interp(f, [i], vec=True)
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3),(25,2.5,0.3)])
-    def test_list_vectorize(self,i,v1,v2):
-        myjitdriver = JitDriver(greens = [],
-                                reds = 'auto')
-        class ListF(object):
-            def __init__(self, size, init):
-                self.list = [init] * size
-            def __getitem__(self, key):
-                return self.list[key]
-            def __setitem__(self, key, value):
-                self.list[key] = value
-        def f(d, v1, v2):
-            a = ListF(d, v1)
-            b = ListF(d, v2)
-            i = 0
-            while i < d:
-                myjitdriver.jit_merge_point()
-                a[i] = a[i] + b[i]
-                i += 1
-            s = 0
-            for i in range(d):
-                s += a[i]
-            return s
-        res = self.meta_interp(f, [i,v1,v2], vec=True, vec_all=True)
-        # sum helps to generate the rounding error of floating points
-        # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
-        assert res == f(i,v1,v2) == sum([v1+v2]*i)
 
     @py.test.mark.parametrize('size',[12])
     def test_body_multiple_accesses(self, size):
@@ -880,17 +843,18 @@
                 j += 4
                 i += 8
 
-        va = alloc_raw_storage(4*30, zero=True)
-        vb = alloc_raw_storage(8*30, zero=True)
-        for i,v in enumerate([1]*30):
+        count = 32
+        va = alloc_raw_storage(4*count, zero=True)
+        vb = alloc_raw_storage(8*count, zero=True)
+        for i,v in enumerate([1,2,3,4]*(count/4)):
             raw_storage_setitem(va, i*4, rffi.cast(rffi.INT,v))
-        for i,v in enumerate([-9.0]*30):
+        for i,v in enumerate([-1.0,-2.0,-3.0,-4.0]*(count/4)):
             raw_storage_setitem(vb, i*8, rffi.cast(rffi.DOUBLE,v))
-        vc = alloc_raw_storage(8*30, zero=True)
-        self.meta_interp(f, [8*30, va, vb, vc], vec=True)
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
 
-        for i in range(30):
-            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == -8.0
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
 
         free_raw_storage(va)
         free_raw_storage(vb)
@@ -898,4 +862,14 @@
 
 
 class TestLLtype(LLJitMixin, VectorizeTests):
-    pass
+    # skip some tests on this backend
+    def test_unpack_f(self):
+        pass
+    def test_unpack_i64(self):
+        pass
+    def test_unpack_i(self):
+        pass
+    def test_unpack_several(self):
+        pass
+    def test_vec_int_sum(self):
+        pass
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -323,50 +323,48 @@
 
     def __add__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x + y)
     __radd__ = __add__
 
     def __sub__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x - y)
 
     def __rsub__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # may be a float
         return self._widen(other, x - y)
 
     def __mul__(self, other):
         x = long(self)
-        if not isinstance(other, (int, long)):
-            return x * other
-        y = long(other)
+        y = other      # may be a float
         return self._widen(other, x * y)
     __rmul__ = __mul__
 
     def __div__(self, other):
         x = long(self)
-        y = long(other)
-        return self._widen(other, x // y)
+        y = other      # may be a float
+        return self._widen(other, x / y)
 
     __floordiv__ = __div__
 
     def __rdiv__(self, other):
         y = long(self)
-        x = long(other)
-        return self._widen(other, x // y)
+        x = other      # may be a float
+        return self._widen(other, x / y)
 
     __rfloordiv__ = __rdiv__
 
     def __mod__(self, other):
         x = long(self)
-        y = long(other)
+        y = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __rmod__(self, other):
         y = long(self)
-        x = long(other)
+        x = other      # not rpython if it is a float
         return self._widen(other, x % y)
 
     def __divmod__(self, other):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -679,21 +679,22 @@
         raise OSError(error, "readdir failed")
     return result
 
-def fdlistdir(dirfd):
-    """
-    Like listdir(), except that the directory is specified as an open
-    file descriptor.
+if not _WIN32:
+    def fdlistdir(dirfd):
+        """
+        Like listdir(), except that the directory is specified as an open
+        file descriptor.
 
-    Note: fdlistdir() closes the file descriptor.  To emulate the
-    Python 3.x 'os.opendir(dirfd)', you must first duplicate the
-    file descriptor.
-    """
-    dirp = c_fdopendir(dirfd)
-    if not dirp:
-        error = get_saved_errno()
-        c_close(dirfd)
-        raise OSError(error, "opendir failed")
-    return _listdir(dirp, rewind=True)
+        Note: fdlistdir() closes the file descriptor.  To emulate the
+        Python 3.x 'os.opendir(dirfd)', you must first duplicate the
+        file descriptor.
+        """
+        dirp = c_fdopendir(dirfd)
+        if not dirp:
+            error = get_saved_errno()
+            c_close(dirfd)
+            raise OSError(error, "opendir failed")
+        return _listdir(dirp, rewind=True)
 
 @replace_os_function('listdir')
 @specialize.argtype(0)
@@ -1166,6 +1167,10 @@
 @replace_os_function('pipe')
 def pipe(flags=0):
     # 'flags' might be ignored.  Check the result.
+    # The handles returned are always inheritable on Posix.
+    # The situation on Windows is not completely clear: I think
+    # it should always return non-inheritable handles, but CPython
+    # uses SECURITY_ATTRIBUTES to ensure that and we don't.
     if _WIN32:
         # 'flags' ignored
         ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1)
@@ -2146,8 +2151,9 @@
         handle_posix_error('mknodat', error)
 
 
-eci_inheritable = eci.merge(ExternalCompilationInfo(
-    separate_module_sources=[r"""
+if not _WIN32:
+    eci_inheritable = eci.merge(ExternalCompilationInfo(
+        separate_module_sources=[r"""
 #include <errno.h>
 #include <sys/ioctl.h>
 
@@ -2200,10 +2206,6 @@
 RPY_EXTERN
 int rpy_dup_noninheritable(int fd)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUPFD_CLOEXEC
     return fcntl(fd, F_DUPFD_CLOEXEC, 0);
 #else
@@ -2221,10 +2223,6 @@
 RPY_EXTERN
 int rpy_dup2_noninheritable(int fd, int fd2)
 {
-#ifdef _WIN32
-#error NotImplementedError
-#endif
-
 #ifdef F_DUP2FD_CLOEXEC
     return fcntl(fd, F_DUP2FD_CLOEXEC, fd2);
 
@@ -2249,33 +2247,41 @@
     return 0;
 #endif
 }
-    """ % {'HAVE_DUP3': HAVE_DUP3}],
-    post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
-                       'RPY_EXTERN int rpy_get_inheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
-                       'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n']))
+        """ % {'HAVE_DUP3': HAVE_DUP3}],
+        post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n'
+                           'RPY_EXTERN int rpy_get_inheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup_noninheritable(int);\n'
+                           'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n'
+                           ]))
 
-c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
-                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                             compilation_info=eci_inheritable)
-c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
-                                rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                compilation_info=eci_inheritable)
-c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
-                                 rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
-                                 compilation_info=eci_inheritable)
+    _c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    _c_get_inheritable = external('rpy_get_inheritable', [rffi.INT],
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                  compilation_info=eci_inheritable)
+    c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT],
+                                    rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                    compilation_info=eci_inheritable)
+    c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT],
+                                     rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,
+                                     compilation_info=eci_inheritable)
 
-def set_inheritable(fd, inheritable):
-    result = c_set_inheritable(fd, inheritable)
-    handle_posix_error('set_inheritable', result)
+    def set_inheritable(fd, inheritable):
+        result = _c_set_inheritable(fd, inheritable)
+        handle_posix_error('set_inheritable', result)
 
-def get_inheritable(fd):
-    res = c_get_inheritable(fd)
-    res = handle_posix_error('get_inheritable', res)
-    return res != 0
+    def get_inheritable(fd):
+        res = _c_get_inheritable(fd)
+        res = handle_posix_error('get_inheritable', res)
+        return res != 0
+
+else:
+    # _WIN32
+    from rpython.rlib.rwin32 import set_inheritable, get_inheritable
+    from rpython.rlib.rwin32 import c_dup_noninheritable
+    from rpython.rlib.rwin32 import c_dup2_noninheritable
+
 
 class SetNonInheritableCache(object):
     """Make one prebuilt instance of this for each path that creates
@@ -2315,20 +2321,20 @@
 
 _pipe2_syscall = ENoSysCache()
 
-post_include_bits=['RPY_EXTERN int _cpu_count(void);']
+post_include_bits=['RPY_EXTERN int rpy_cpu_count(void);']
 # cpu count for linux, windows and mac (+ bsds)
 # note that the code is copied from cpython and split up here
 if sys.platform.startswith('linux'):
     cpucount_eci = ExternalCompilationInfo(includes=["unistd.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 return sysconf(_SC_NPROCESSORS_ONLN);
             }
             """], post_include_bits=post_include_bits)
 elif sys.platform == "win32":
     cpucount_eci = ExternalCompilationInfo(includes=["Windows.h"],
             separate_module_sources=["""
-        RPY_EXTERN int _cpu_count(void) {
+        RPY_EXTERN int rpy_cpu_count(void) {
             SYSTEM_INFO sysinfo;
             GetSystemInfo(&sysinfo);
             return sysinfo.dwNumberOfProcessors;
@@ -2337,7 +2343,7 @@
 else:
     cpucount_eci = ExternalCompilationInfo(includes=["sys/types.h", "sys/sysctl.h"],
             separate_module_sources=["""
-            RPY_EXTERN int _cpu_count(void) {
+            RPY_EXTERN int rpy_cpu_count(void) {
                 int ncpu = 0;
             #if defined(__DragonFly__) || \
                 defined(__OpenBSD__)   || \
@@ -2355,7 +2361,7 @@
             }
             """], post_include_bits=post_include_bits)
 
-_cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real,
+_cpu_count = rffi.llexternal('rpy_cpu_count', [], rffi.INT_real,
                             compilation_info=cpucount_eci)
 
 def cpu_count():
diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py
--- a/rpython/rlib/rposix_environ.py
+++ b/rpython/rlib/rposix_environ.py
@@ -201,6 +201,8 @@
     # default implementation for platforms without a real unsetenv()
     r_putenv(name, '')
 
+REAL_UNSETENV = False
+
 if hasattr(__import__(os.name), 'unsetenv'):
     os_unsetenv = rffi.llexternal('unsetenv', [rffi.CCHARP], rffi.INT,
                                   save_err=rffi.RFFI_SAVE_ERRNO)
@@ -222,3 +224,4 @@
     register_external(r_unsetenv, [str0], annmodel.s_None,
                       export_name='ll_os.ll_os_unsetenv',
                       llimpl=unsetenv_llimpl)
+    REAL_UNSETENV = True
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -641,6 +641,7 @@
 
     @specialize.arg(0)
     def win32_xstat(traits, path, traverse=False):
+        # XXX 'traverse' is ignored
         win32traits = make_win32_traits(traits)
         with lltype.scoped_alloc(
                 win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data:
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1062,11 +1062,32 @@
     return result
 make_socket._annspecialcase_ = 'specialize:arg(4)'
 
-def sock_set_inheritable(fd, inheritable):
-    try:
-        rposix.set_inheritable(fd, inheritable)
-    except OSError as e:
-        raise CSocketError(e.errno)
+if _c.WIN32:
+    def sock_set_inheritable(fd, inheritable):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            rwin32.set_handle_inheritable(handle, inheritable)
+        except WindowsError:
+            raise RSocketError("SetHandleInformation failed")   # xxx
+
+    def sock_get_inheritable(fd):
+        handle = rffi.cast(rwin32.HANDLE, fd)
+        try:
+            return rwin32.get_handle_inheritable(handle)
+        except WindowsError:
+            raise RSocketError("GetHandleInformation failed")   # xxx
+else:
+    def sock_set_inheritable(fd, inheritable):
+        try:
+            rposix.set_inheritable(fd, inheritable)
+        except OSError as e:
+            raise CSocketError(e.errno)
+
+    def sock_get_inheritable(fd):
+        try:
+            return rposix.get_inheritable(fd)
+        except OSError as e:
+            raise CSocketError(e.errno)
 
 class SocketError(Exception):
     applevelerrcls = 'error'
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -112,7 +112,7 @@
                        CTRL_C_EVENT CTRL_BREAK_EVENT
                        MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION
                        WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE
-                       STD_ERROR_HANDLE
+                       STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR
                     """
         from rpython.translator.platform import host_factory
         static_platform = host_factory()
@@ -473,3 +473,64 @@
     CONSOLE_SCREEN_BUFFER_INFO_P = lltype.Ptr(CONSOLE_SCREEN_BUFFER_INFO)
     GetConsoleScreenBufferInfo = winexternal(
         "GetConsoleScreenBufferInfo", [HANDLE, CONSOLE_SCREEN_BUFFER_INFO_P], BOOL)
+
+    _GetHandleInformation = winexternal(
+        'GetHandleInformation', [HANDLE, LPDWORD], BOOL)
+    _SetHandleInformation = winexternal(
+        'SetHandleInformation', [HANDLE, DWORD, DWORD], BOOL)
+
+    def set_inheritable(fd, inheritable):
+        handle = get_osfhandle(fd)
+        set_handle_inheritable(handle, inheritable)
+
+    def set_handle_inheritable(handle, inheritable):
+        assert lltype.typeOf(handle) is HANDLE
+        if inheritable:
+            flags = HANDLE_FLAG_INHERIT
+        else:
+            flags = 0
+        if not _SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags):
+            raise lastSavedWindowsError("SetHandleInformation")
+
+    def get_inheritable(fd):
+        handle = get_osfhandle(fd)
+        return get_handle_inheritable(handle)
+
+    def get_handle_inheritable(handle):
+        assert lltype.typeOf(handle) is HANDLE
+        pflags = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
+        try:
+            if not _GetHandleInformation(handle, pflags):
+                raise lastSavedWindowsError("GetHandleInformation")
+            flags = pflags[0]
+        finally:
+            lltype.free(pflags, flavor='raw')
+        return (flags & HANDLE_FLAG_INHERIT) != 0
+
+    _GetFileType = winexternal('GetFileType', [HANDLE], DWORD)
+
+    def c_dup_noninheritable(fd1):
+        from rpython.rlib.rposix import c_dup
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        fd2 = c_dup(fd1)     # the inheritable version
+        if fd2 >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return fd2
+
+    def c_dup2_noninheritable(fd1, fd2):
+        from rpython.rlib.rposix import c_dup2
+
+        ftype = _GetFileType(get_osfhandle(fd1))
+        res = c_dup2(fd1, fd2)     # the inheritable version
+        if res >= 0 and ftype != FILE_TYPE_CHAR:
+            try:
+                set_inheritable(fd2, False)
+            except:
+                os.close(fd2)
+                raise
+        return res
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -602,3 +602,12 @@
 
         assert r_uint64(self._64_umax) + r_uint64(1) == r_uint64(0)
         assert r_uint64(0) - r_uint64(1) == r_uint64(self._64_umax)
+
+    def test_operation_with_float(self):
+        def f(x):
+            assert r_longlong(x) + 0.5 == 43.5
+            assert r_longlong(x) - 0.5 == 42.5
+            assert r_longlong(x) * 0.5 == 21.5
+            assert r_longlong(x) / 0.8 == 53.75
+        f(43)
+        interpret(f, [43])
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -470,11 +470,18 @@
         assert rposix.is_valid_fd(fd) == 0
 
     def test_putenv(self):
+        from rpython.rlib import rposix_environ
+
         def f():
             rposix.putenv(self.path, self.path)
             rposix.unsetenv(self.path)
 
-        interpret(f, []) # does not crash
+        interpret(f, [],     # does not crash
+                  malloc_check=rposix_environ.REAL_UNSETENV)
+        # If we have a real unsetenv(), check that it frees the string
+        # kept alive by putenv().  Otherwise, we can't check that,
+        # because unsetenv() will keep another string alive itself.
+    test_putenv.dont_track_allocations = True
 
 
 class TestPosixAscii(BasePosixUnicodeOrAscii):
@@ -604,6 +611,9 @@
 def test_SetNonInheritableCache():
     cache = rposix.SetNonInheritableCache()
     fd1, fd2 = os.pipe()
+    if sys.platform == 'win32':
+        rposix.set_inheritable(fd1, True)
+        rposix.set_inheritable(fd2, True)
     assert rposix.get_inheritable(fd1) == True
     assert rposix.get_inheritable(fd1) == True
     assert cache.cached_inheritable == -1
@@ -616,6 +626,24 @@
     os.close(fd1)
     os.close(fd2)
 
+def test_dup_dup2_non_inheritable():
+    for preset in [False, True]:
+        fd1, fd2 = os.pipe()
+        rposix.set_inheritable(fd1, preset)
+        rposix.set_inheritable(fd2, preset)
+        fd3 = rposix.dup(fd1, True)
+        assert rposix.get_inheritable(fd3) == True
+        fd4 = rposix.dup(fd1, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd4, False)
+        assert rposix.get_inheritable(fd4) == False
+        rposix.dup2(fd2, fd3, True)
+        assert rposix.get_inheritable(fd3) == True
+        os.close(fd1)
+        os.close(fd2)
+        os.close(fd3)
+        os.close(fd4)
+
 def test_sync():
     if sys.platform != 'win32':
         rposix.sync()
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -124,8 +124,8 @@
         py.test.skip('No socketpair on Windows')
     for inh in [False, True]:
         s1, s2 = socketpair(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
-        assert rposix.get_inheritable(s2.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s2.fd) == inh
         s1.close()
         s2.close()
 
@@ -391,7 +391,7 @@
 def test_inheritable():
     for inh in [False, True]:
         s1 = RSocket(inheritable=inh)
-        assert rposix.get_inheritable(s1.fd) == inh
+        assert sock_get_inheritable(s1.fd) == inh
         s1.close()
 
 def test_getaddrinfo_http():
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -62,7 +62,7 @@
         guard_dict = {}
         for loop_no, loop in enumerate(loops):
             for op in loop.operations:
-                if op.name.startswith('guard_'):
+                if op.name.startswith('guard_') or op.name.startswith('vec_guard_'):
                     guard_dict[int(op.descr[len('<Guard0x'):-1], 16)] = (op, loop)
         for loop in loops:
             if loop.comment:

From pypy.commits at gmail.com  Wed Nov  9 11:19:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 08:19:05 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: updates
Message-ID: <58234c79.c19d1c0a.95cfc.93ac@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5745:dac018501316
Date: 2016-11-09 17:18 +0100
http://bitbucket.org/pypy/extradoc/changeset/dac018501316/

Log:	updates

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -96,4 +96,5 @@
  
 * hash({}.values()) works (but hash({}.keys()) correctly gives
   TypeError).  That's a bit confusing and, as far as I can tell, always
-  pointless.
+  pointless.  Also, related: d.keys()==d.keys() but
+  d.values()!=d.values().
diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,7 @@
 In-progress ("Lock" section)
 ----------------------------
 
-nothing
+* bytes % args, bytearray % args (arigo)
 
 
 Misc stuff not formally in any milestone
@@ -69,7 +69,7 @@
 * The new os.scandir() function (POSIX-DONE, missing Win32)
 
 * Newly created file descriptors are non-inheritable (PEP 446) 
-  (POSIX-DONE, missing Win32)
+  (DONE)
 
 * The marshal format has been made more compact and efficient
   (DONE, maybe a small optimization left---TYPE_*ASCII*---that

From pypy.commits at gmail.com  Wed Nov  9 11:45:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 08:45:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Exhaustive tests for bytes '%' formatting
 (pep461)
Message-ID: <5823529e.d39a1c0a.435b2.194e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88279:fbf931ce560d
Date: 2016-11-09 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/fbf931ce560d/

Log:	Exhaustive tests for bytes '%' formatting (pep461)

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -333,9 +333,102 @@
         assert "<%r>" % "\xe9" == "<'\xe9'>"
         assert "<%a>" % "\xe9" == "<'\\xe9'>"
 
+
+class AppTestBytes:
+
     def test_ascii_bytes(self):
         assert b"<%a>" % b"test" == b"<b'test'>"
         assert b"<%a>" % b"\t\x80" == b"<b'\\t\\x80'>"
         assert repr(b"\xe9") == "b'\\xe9'"
+        assert b"<%a>" % b"\xe9" == b"<b'\\xe9'>"
+        assert b"<%a>" % "foo" == b"<'foo'>"
+        assert b"<%a>" % "\u1234" == b"<'\\u1234'>"
+
+    def test_r_compat_bytes(self):
+        assert b"<%r>" % b"test" == b"<b'test'>"
+        assert b"<%r>" % b"\t\x80" == b"<b'\\t\\x80'>"
+        assert repr(b"\xe9") == "b'\\xe9'"
         assert b"<%r>" % b"\xe9" == b"<b'\\xe9'>"
-        assert b"<%a>" % b"\xe9" == b"<b'\\xe9'>"
+        assert b"<%r>" % "foo" == b"<'foo'>"
+        assert b"<%r>" % "\u1234" == b"<'\\u1234'>"
+
+    def test_numeric_bytes(self):
+        assert b"<%4x>" % 10 == b"<   a>"
+        assert b"<%#4x>" % 10 == b"< 0xa>"
+        assert b"<%04X>" % 10 == b"<000A>"
+
+    def test_char_bytes(self):
+        assert b"<%c>" % 48 == b"<0>"
+        assert b"<%c>" % b"?" == b"<?>"
+        raises(TypeError, 'b"<%c>" % "?"')
+
+    def test_bytes_bytes(self):
+        assert b"<%b>" % b"123" == b"<123>"
+        class Foo:
+            def __bytes__(self):
+                return b"123"
+        assert b"<%b>" % Foo() == b"<123>"
+        raises(TypeError, 'b"<%b>" % 42')
+        raises(TypeError, 'b"<%b>" % "?"')
+
+    def test_s_compat_bytes(self):
+        assert b"<%s>" % b"123" == b"<123>"
+        class Foo:
+            def __bytes__(self):
+                return b"123"
+        assert b"<%s>" % Foo() == b"<123>"
+        raises(TypeError, 'b"<%s>" % 42')
+        raises(TypeError, 'b"<%s>" % "?"')
+
+
+class AppTestBytearray:
+
+    def test_ascii_bytes(self):
+        assert bytearray(b"<%a>") % b"test" == bytearray(b"<b'test'>")
+        assert bytearray(b"<%a>") % b"\t\x80" == bytearray(b"<b'\\t\\x80'>")
+        assert repr(b"\xe9") == "b'\\xe9'"
+        assert bytearray(b"<%a>") % b"\xe9" == bytearray(b"<b'\\xe9'>")
+        assert bytearray(b"<%a>") % "foo" == bytearray(b"<'foo'>")
+        assert bytearray(b"<%a>") % "\u1234" == bytearray(b"<'\\u1234'>")
+
+    def test_bytearray_not_modified(self):
+        b1 = bytearray(b"<%a>")
+        b2 = b1 % b"test"
+        assert b1 == bytearray(b"<%a>")
+        assert b2 == bytearray(b"<b'test'>")
+
+    def test_r_compat_bytes(self):
+        assert bytearray(b"<%r>") % b"test" == bytearray(b"<b'test'>")
+        assert bytearray(b"<%r>") % b"\t\x80" == bytearray(b"<b'\\t\\x80'>")
+        assert repr(b"\xe9") == "b'\\xe9'"
+        assert bytearray(b"<%r>") % b"\xe9" == bytearray(b"<b'\\xe9'>")
+        assert bytearray(b"<%r>") % "foo" == bytearray(b"<'foo'>")
+        assert bytearray(b"<%r>") % "\u1234" == bytearray(b"<'\\u1234'>")
+
+    def test_numeric_bytes(self):
+        assert bytearray(b"<%4x>") % 10 == bytearray(b"<   a>")
+        assert bytearray(b"<%#4x>") % 10 == bytearray(b"< 0xa>")
+        assert bytearray(b"<%04X>") % 10 == bytearray(b"<000A>")
+
+    def test_char_bytes(self):
+        assert bytearray(b"<%c>") % 48 == bytearray(b"<0>")
+        assert bytearray(b"<%c>") % b"?" == bytearray(b"<?>")
+        raises(TypeError, 'bytearray(b"<%c>") % "?"')
+
+    def test_bytes_bytes(self):
+        assert bytearray(b"<%b>") % b"123" == bytearray(b"<123>")
+        class Foo:
+            def __bytes__(self):
+                return b"123"
+        assert bytearray(b"<%b>") % Foo() == bytearray(b"<123>")
+        raises(TypeError, 'bytearray(b"<%b>") % 42')
+        raises(TypeError, 'bytearray(b"<%b>") % "?"')
+
+    def test_s_compat_bytes(self):
+        assert bytearray(b"<%s>") % b"123" == bytearray(b"<123>")
+        class Foo:
+            def __bytes__(self):
+                return b"123"
+        assert bytearray(b"<%s>") % Foo() == bytearray(b"<123>")
+        raises(TypeError, 'bytearray(b"<%s>") % 42')
+        raises(TypeError, 'bytearray(b"<%s>") % "?"')

From pypy.commits at gmail.com  Wed Nov  9 11:45:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 08:45:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix for %s on bytes,
 otherwise it already works
Message-ID: <582352a0.c1341c0a.942da.6dd8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88280:fd7154e68fb8
Date: 2016-11-09 17:44 +0100
http://bitbucket.org/pypy/pypy/changeset/fd7154e68fb8/

Log:	Fix for %s on bytes, otherwise it already works

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -422,20 +422,18 @@
             return space.str_w(w_result)
 
         def fmt_s(self, w_value):
+            if not do_unicode:
+                # on bytes, %s is equivalent to %b
+                self.fmt_b(w_value)
+                return
             space = self.space
-            got_unicode = space.isinstance_w(w_value,
-                                                         space.w_unicode)
-            if not do_unicode:
-                if got_unicode:
-                    raise NeedUnicodeFormattingError
-                s = self.string_formatting(w_value)
+            got_unicode = space.isinstance_w(w_value, space.w_unicode)
+            if not got_unicode:
+                w_value = space.call_function(space.w_unicode, w_value)
             else:
-                if not got_unicode:
-                    w_value = space.call_function(space.w_unicode, w_value)
-                else:
-                    from pypy.objspace.std.unicodeobject import unicode_from_object
-                    w_value = unicode_from_object(space, w_value)
-                s = space.unicode_w(w_value)
+                from pypy.objspace.std.unicodeobject import unicode_from_object
+                w_value = unicode_from_object(space, w_value)
+            s = space.unicode_w(w_value)
             self.std_wp(s)
 
         def fmt_r(self, w_value):

From pypy.commits at gmail.com  Wed Nov  9 12:11:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 09:11:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Tweak the tests until they pass -A
Message-ID: <582358b0.c11d1c0a.55bc4.7b40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88281:6c7cc24525c5
Date: 2016-11-09 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/6c7cc24525c5/

Log:	Tweak the tests until they pass -A

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -80,7 +80,7 @@
     def test_format_float(self):
         f = 23.456
         assert '23' == '%d' % f
-        assert '17' == '%x' % f
+        raises(TypeError, "'%x' % f")
         assert '23.456' == '%s' % f
         # for 'r' use a float that has an exact decimal rep:
         g = 23.125
@@ -175,29 +175,31 @@
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))
 
-    def test___int__(self):
+    def test___int__index__(self):
         class MyInt(object):
             def __init__(self, x):
                 self.x = x
             def __int__(self):
                 return self.x
-        x = MyInt(65)
-        assert '%c' % x == 'A'
+        x = MyInt(33)
+        raises(TypeError, "'%c' % x")
+        MyInt.__index__ = lambda self: self.x * 2
+        assert '%c' % x == 'B'
 
-    def test_int_fails(self):
-        class IntFails(object):
-            def __int__(self):
+    def test_index_fails(self):
+        class IndexFails(object):
+            def __index__(self):
                 raise Exception
 
-        exc = raises(TypeError, "%x".__mod__, IntFails())
-        expected = "%x format: a number is required, not IntFails"
+        exc = raises(TypeError, "%x".__mod__, IndexFails())
+        expected = "%x format: an integer is required, not IndexFails"
         assert str(exc.value) == expected
 
     def test_formatting_huge_precision(self):
         prec = 2**31
         format_string = "%.{}f".format(prec)
         exc = raises(ValueError, "format_string % 2.34")
-        assert str(exc.value) == 'prec too big'
+        assert str(exc.value) == 'precision too big'
         raises(OverflowError, lambda: u'%.*f' % (prec, 1. / 7))
 
     def test_formatting_huge_width(self):
@@ -317,7 +319,7 @@
         prec = 2**31
         format_string = u"%.{}f".format(prec)
         exc = raises(ValueError, "format_string % 2.34")
-        assert str(exc.value) == 'prec too big'
+        assert str(exc.value) == 'precision too big'
         raises(OverflowError, lambda: u'%.*f' % (prec, 1. / 7))
 
     def test_formatting_huge_width(self):

From pypy.commits at gmail.com  Wed Nov  9 12:11:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 09:11:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Small tweaks until the tests pass on pypy
 too
Message-ID: <582358b2.2636c20a.6c87.3674@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88282:254669bd5f21
Date: 2016-11-09 18:10 +0100
http://bitbucket.org/pypy/pypy/changeset/254669bd5f21/

Log:	Small tweaks until the tests pass on pypy too

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -213,7 +213,7 @@
 
             if self.peekchr() == '.':
                 self.forward()
-                self.prec = self.peel_num('prec', INT_MAX)
+                self.prec = self.peel_num('precision', INT_MAX)
                 if self.prec < 0:
                     self.prec = 0    # this can happen:  '%.*f' % (-5, 3)
             else:
@@ -259,7 +259,7 @@
                 w_value = self.nextinputvalue()
                 if name == 'width':
                     return space.int_w(w_value)
-                elif name == 'prec':
+                elif name == 'precision':
                     return space.c_int_w(w_value)
                 else:
                     assert False
@@ -460,18 +460,12 @@
         def fmt_c(self, w_value):
             self.prec = -1     # just because
             space = self.space
-            if space.isinstance_w(w_value, space.w_str):
-                s = space.str_w(w_value)
-                if len(s) != 1:
-                    raise oefmt(space.w_TypeError, "%c requires int or char")
-                self.std_wp(s)
-            elif space.isinstance_w(w_value, space.w_unicode):
-                if not do_unicode:
-                    raise NeedUnicodeFormattingError
-                ustr = space.unicode_w(w_value)
-                if len(ustr) != 1:
-                    raise oefmt(space.w_TypeError, "%c requires int or unichar")
-                self.std_wp(ustr)
+            try:
+                w_value = space.index(w_value)
+            except OperationError as e:
+                if e.async(space):
+                    raise
+                # otherwise, eats all exceptions, like CPython
             else:
                 n = space.int_w(w_value)
                 if do_unicode:
@@ -488,6 +482,23 @@
                         raise oefmt(space.w_OverflowError,
                                     "character code not in range(256)")
                     self.std_wp(s)
+                return
+            if space.isinstance_w(w_value, space.w_str):
+                s = space.str_w(w_value)
+                if len(s) == 1:
+                    self.std_wp(s)
+                    return
+            elif space.isinstance_w(w_value, space.w_unicode):
+                if not do_unicode:
+                    raise NeedUnicodeFormattingError
+                ustr = space.unicode_w(w_value)
+                if len(ustr) == 1:
+                    self.std_wp(ustr)
+                    return
+            if do_unicode:
+                raise oefmt(space.w_TypeError, "%c requires int or char")
+            else:
+                raise oefmt(space.w_TypeError, "%c requires int or single byte")
 
         def fmt_b(self, w_value):
             space = self.space
@@ -581,18 +592,22 @@
     # make sure that w_value is a wrapped integer
     return space.int(w_value)
 
+def maybe_index(space, w_value):
+    return space.index(w_value)
+
 def maybe_float(space, w_value):
     # make sure that w_value is a wrapped float
     return space.float(w_value)
 
-def format_num_helper_generator(fmt, digits):
+def format_num_helper_generator(fmt, digits, decoder=maybe_int,
+                                expect_text="a number"):
     def format_num_helper(space, w_value):
         try:
-            w_value = maybe_int(space, w_value)
+            w_value = decoder(space, w_value)
         except OperationError:
             raise oefmt(space.w_TypeError,
-                        "%s format: a number is required, not %T",
-                        fmt, w_value)
+                        "%s format: %s is required, not %T",
+                        fmt, expect_text, w_value)
         try:
             value = space.int_w(w_value)
             return fmt % (value,)
@@ -605,5 +620,7 @@
                               'base%d_num_helper' % len(digits))
 
 int_num_helper = format_num_helper_generator('%d', '0123456789')
-oct_num_helper = format_num_helper_generator('%o', '01234567')
-hex_num_helper = format_num_helper_generator('%x', '0123456789abcdef')
+oct_num_helper = format_num_helper_generator('%o', '01234567',
+                     decoder=maybe_index, expect_text="an integer")
+hex_num_helper = format_num_helper_generator('%x', '0123456789abcdef',
+                     decoder=maybe_index, expect_text="an integer")
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -78,10 +78,14 @@
         raises(TypeError, '%d'.__mod__, s)
 
     def test_format_float(self):
-        f = 23.456
-        assert '23' == '%d' % f
+        f = -23.456
+        assert '-23' == '%d' % f
+        assert '-23' == '%i' % f
+        assert '-23' == '%u' % f
         raises(TypeError, "'%x' % f")
-        assert '23.456' == '%s' % f
+        raises(TypeError, "'%X' % f")
+        raises(TypeError, "'%o' % f")
+        assert '-23.456' == '%s' % f
         # for 'r' use a float that has an exact decimal rep:
         g = 23.125
         assert '23.125' == '%r' % g
@@ -194,6 +198,7 @@
         exc = raises(TypeError, "%x".__mod__, IndexFails())
         expected = "%x format: an integer is required, not IndexFails"
         assert str(exc.value) == expected
+        raises(TypeError, "%c".__mod__, IndexFails())
 
     def test_formatting_huge_precision(self):
         prec = 2**31

From pypy.commits at gmail.com  Wed Nov  9 12:11:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 Nov 2016 09:11:21 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: PEP 461 Done
Message-ID: <582358b9.0370c20a.98ff1.3bcc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5746:a5c3f0331c0e
Date: 2016-11-09 18:11 +0100
http://bitbucket.org/pypy/extradoc/changeset/a5c3f0331c0e/

Log:	PEP 461 Done

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,7 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* bytes % args, bytearray % args (arigo)
+(empty)
 
 
 Misc stuff not formally in any milestone
@@ -57,7 +57,7 @@
 
 * PEP 448, additional unpacking generalizations.
 
-* bytes % args, bytearray % args: PEP 461
+* bytes % args, bytearray % args: PEP 461 (DONE)
 
 * New bytes.hex(), bytearray.hex() and memoryview.hex() methods. (DONE)
 

From pypy.commits at gmail.com  Wed Nov  9 13:16:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 09 Nov 2016 10:16:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Disable SSL part of the test for now
 (times out)
Message-ID: <58236814.e626c20a.c4c66.58f5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88283:d267f0decb98
Date: 2016-11-09 18:16 +0000
http://bitbucket.org/pypy/pypy/changeset/d267f0decb98/

Log:	Disable SSL part of the test for now (times out)

diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -1665,7 +1665,7 @@
         logger = logging.getLogger("http")
         root_logger = self.root_logger
         root_logger.removeHandler(self.root_logger.handlers[0])
-        for secure in (False, True):
+        for secure in (False,):  # XXX: disable SSL tests until it works
             addr = ('localhost', 0)
             if secure:
                 try:

From pypy.commits at gmail.com  Thu Nov 10 02:55:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 09 Nov 2016 23:55:02 -0800 (PST)
Subject: [pypy-commit] pypy default: add paragraph to release notes (vecopt)
Message-ID: <582427d6.0209c20a.8f773.3a4e@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r88284:16700fced491
Date: 2016-11-10 08:54 +0100
http://bitbucket.org/pypy/pypy/changeset/16700fced491/

Log:	add paragraph to release notes (vecopt)

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -11,7 +11,10 @@
 
 Work proceeds at a good pace on the PyPy3.5
 version due to a grant_ from the Mozilla Foundation, and some of those
-changes have been backported to PyPy2.7 where relevant
+changes have been backported to PyPy2.7 where relevant.
+
+The PowerPC and s390x backend have been enhanced_ with the capability use SIMD instructions
+ for micronumpy loops.
 
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
@@ -45,6 +48,7 @@
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+.. _`enhanced`: https://morepypy.blogspot.co.at/2016/11/vectorization-extended-powerpc-and-s390x.html
 
 What is PyPy?
 =============

From pypy.commits at gmail.com  Thu Nov 10 03:46:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 00:46:49 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <582433f9.0a4cc20a.3a027.48b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88285:76a9a43a1e8c
Date: 2016-11-10 09:45 +0100
http://bitbucket.org/pypy/pypy/changeset/76a9a43a1e8c/

Log:	fix test

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -340,7 +340,10 @@
     def __mul__(self, other):
         x = long(self)
         y = other      # may be a float
-        return self._widen(other, x * y)
+        z = x * y
+        if isinstance(z, (int, long)):
+            z = self._widen(other, z)
+        return z
     __rmul__ = __mul__
 
     def __div__(self, other):

From pypy.commits at gmail.com  Thu Nov 10 03:46:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 00:46:50 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <582433fa.8f95c20a.34d91.53da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88286:0b074a6a6472
Date: 2016-11-10 09:46 +0100
http://bitbucket.org/pypy/pypy/changeset/0b074a6a6472/

Log:	merge heads

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -340,7 +340,10 @@
     def __mul__(self, other):
         x = long(self)
         y = other      # may be a float
-        return self._widen(other, x * y)
+        z = x * y
+        if isinstance(z, (int, long)):
+            z = self._widen(other, z)
+        return z
     __rmul__ = __mul__
 
     def __div__(self, other):

From pypy.commits at gmail.com  Thu Nov 10 03:59:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 00:59:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix error message
Message-ID: <582436e5.031f1c0a.b0c7f.ce13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88288:3fff80d67303
Date: 2016-11-10 09:58 +0100
http://bitbucket.org/pypy/pypy/changeset/3fff80d67303/

Log:	fix error message

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -716,9 +716,15 @@
                 if mytype.method != '' and e.match(space, space.w_TypeError):
                     try:
                         item = unwrap(space.call_method(w_item, mytype.method))
-                    except OperationError:
-                        raise oefmt(space.w_TypeError,
-                                    "array item must be " + mytype.unwrap[:-2])
+                    except OperationError as e:
+                        if e.async(space):
+                            raise
+                        if space.isinstance_w(w_item, space.w_unicode):
+                            msg = ("cannot use a str to initialize an array"
+                                   " with typecode '" + mytype.typecode + "'")
+                        else:
+                            msg = "array item must be " + mytype.unwrap[:-2]
+                        raise OperationError(space.w_TypeError, space.wrap(msg))
                 else:
                     raise
             if mytype.convert:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -1158,3 +1158,9 @@
         it = iter(array.array('b'))
         assert list(it) == []
         assert list(iter(it)) == []
+
+    def test_array_cannot_use_str(self):
+        import array
+        e = raises(TypeError, array.array, 'i', 'abcd')
+        assert str(e.value) == ("cannot use a str to initialize an array"
+                                " with typecode 'i'")

From pypy.commits at gmail.com  Thu Nov 10 03:59:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 00:59:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for test
Message-ID: <582436e3.cf3fc20a.1e77d.5604@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88287:01bd875e7563
Date: 2016-11-10 09:58 +0100
http://bitbucket.org/pypy/pypy/changeset/01bd875e7563/

Log:	fix for test

diff --git a/pypy/module/math/app_math.py b/pypy/module/math/app_math.py
--- a/pypy/module/math/app_math.py
+++ b/pypy/module/math/app_math.py
@@ -1,5 +1,8 @@
 import sys
-from _operator import index
+try:
+    from _operator import index
+except ImportError:
+    pass      # for tests only
 
 def factorial(x):
     """factorial(x) -> Integral

From pypy.commits at gmail.com  Thu Nov 10 04:39:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 01:39:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: cmath.isclose()
Message-ID: <58244034.c6bdc20a.e3d52.64d0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88289:a7657f5b543d
Date: 2016-11-10 10:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a7657f5b543d/

Log:	cmath.isclose()

diff --git a/pypy/module/cmath/__init__.py b/pypy/module/cmath/__init__.py
--- a/pypy/module/cmath/__init__.py
+++ b/pypy/module/cmath/__init__.py
@@ -40,6 +40,7 @@
     interpleveldefs = {
         'pi': 'space.wrap(interp_cmath.pi)',
         'e':  'space.wrap(interp_cmath.e)',
+        'isclose': 'interp_cmath.isclose',
     }
     interpleveldefs.update(dict([(name, 'interp_cmath.wrapped_' + name)
                                  for name in names_and_docstrings]))
diff --git a/pypy/module/cmath/interp_cmath.py b/pypy/module/cmath/interp_cmath.py
--- a/pypy/module/cmath/interp_cmath.py
+++ b/pypy/module/cmath/interp_cmath.py
@@ -2,8 +2,9 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.cmath import names_and_docstrings
-from rpython.rlib import rcomplex
+from rpython.rlib import rcomplex, rfloat
 
 pi = math.pi
 e  = math.e
@@ -174,3 +175,54 @@
     res = c_isfinite(x, y)
     return space.newbool(res)
 wrapped_isfinite.func_doc = names_and_docstrings['isfinite']
+
+
+ at unwrap_spec(rel_tol=float, abs_tol=float)
+def isclose(space, w_a, w_b, __kwonly__, rel_tol=1e-09, abs_tol=0.0):
+    """isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) -> bool
+
+Determine whether two complex numbers are close in value.
+
+   rel_tol
+       maximum difference for being considered "close", relative to the
+       magnitude of the input values
+   abs_tol
+       maximum difference for being considered "close", regardless of the
+       magnitude of the input values
+
+Return True if a is close in value to b, and False otherwise.
+
+For the values to be considered close, the difference between them
+must be smaller than at least one of the tolerances.
+
+-inf, inf and NaN behave similarly to the IEEE 754 Standard.  That
+is, NaN is not close to anything, even itself.  inf and -inf are
+only close to themselves."""
+    ax, ay = space.unpackcomplex(w_a)
+    bx, by = space.unpackcomplex(w_b)
+    #
+    # sanity check on the inputs
+    if rel_tol < 0.0 or abs_tol < 0.0:
+        raise oefmt(space.w_ValueError, "tolerances must be non-negative")
+    #
+    # short circuit exact equality -- needed to catch two infinities of
+    # the same sign. And perhaps speeds things up a bit sometimes.
+    if ax == bx and ay == by:
+        return space.w_True
+    #
+    # This catches the case of two infinities of opposite sign, or
+    # one infinity and one finite number. Two infinities of opposite
+    # sign would otherwise have an infinite relative tolerance.
+    # Two infinities of the same sign are caught by the equality check
+    # above.
+    if (rfloat.isinf(ax) or rfloat.isinf(ay) or
+        rfloat.isinf(bx) or rfloat.isinf(by)):
+        return space.w_False
+    #
+    # now do the regular computation
+    # this is essentially the "weak" test from the Boost library
+    diff = c_abs(bx - ax, by - ay)
+    result = ((diff <= rel_tol * c_abs(bx, by) or
+               diff <= rel_tol * c_abs(ax, ay)) or
+              diff <= abs_tol)
+    return space.newbool(result)
diff --git a/pypy/module/cmath/test/test_cmath.py b/pypy/module/cmath/test/test_cmath.py
--- a/pypy/module/cmath/test/test_cmath.py
+++ b/pypy/module/cmath/test/test_cmath.py
@@ -118,6 +118,22 @@
                 return 2.0
         assert cmath.polar(Foo()) == (2, 0)
 
+    def test_isclose(self):
+        import cmath
+        raises(ValueError, cmath.isclose, 2, 3, rel_tol=-0.5)
+        raises(ValueError, cmath.isclose, 2, 3, abs_tol=-0.5)
+        for z in [0.0, 1.0, 1j,
+                  complex("inf"), complex("infj"),
+                  complex("-inf"), complex("-infj")]:
+            assert cmath.isclose(z, z)
+        assert not cmath.isclose(complex("infj"), complex("-infj"))
+        assert cmath.isclose(1j, 1j+1e-12)
+        assert not cmath.isclose(1j, 1j+1e-12, rel_tol=1e-13)
+        assert not cmath.isclose(100000j, 100001j)
+        assert cmath.isclose(100000j, 100001j, rel_tol=1e-4)
+        assert cmath.isclose(100000j, 100001j, abs_tol=1.5)
+        assert not cmath.isclose(100000j, 100001j, abs_tol=0.5)
+
 
 def parse_testfile(fname):
     """Parse a file with test values

From pypy.commits at gmail.com  Thu Nov 10 04:48:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 01:48:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: accept AttributeError in this case too
Message-ID: <58244286.4438c20a.71ee3.6756@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88290:db1c76818220
Date: 2016-11-10 10:48 +0100
http://bitbucket.org/pypy/pypy/changeset/db1c76818220/

Log:	accept AttributeError in this case too

diff --git a/lib-python/3/test/test_generators.py b/lib-python/3/test/test_generators.py
--- a/lib-python/3/test/test_generators.py
+++ b/lib-python/3/test/test_generators.py
@@ -96,8 +96,8 @@
         # generator names must be a string and cannot be deleted
         self.assertRaises(TypeError, setattr, gen, '__name__', 123)
         self.assertRaises(TypeError, setattr, gen, '__qualname__', 123)
-        self.assertRaises(TypeError, delattr, gen, '__name__')
-        self.assertRaises(TypeError, delattr, gen, '__qualname__')
+        self.assertRaises((TypeError, AttributeError), delattr, gen, '__name__')
+        self.assertRaises((TypeError, AttributeError), delattr, gen, '__qualname__')
 
         # modify names of the function creating the generator
         func.__qualname__ = "func_qualname"

From pypy.commits at gmail.com  Thu Nov 10 04:56:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 01:56:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: sys.is_finalizing()
Message-ID: <58244431.d39a1c0a.435b2.617f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88291:763c24bd5dca
Date: 2016-11-10 10:55 +0100
http://bitbucket.org/pypy/pypy/changeset/763c24bd5dca/

Log:	sys.is_finalizing()

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -443,6 +443,7 @@
         self.wait_for_thread_shutdown()
         w_atexit = self.getbuiltinmodule('atexit')
         self.call_method(w_atexit, '_run_exitfuncs')
+        self.sys.finalizing = True
         self.sys.flush_std_files(self)
         from pypy.interpreter.module import Module
         for w_mod in self.builtin_modules.values():
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -27,6 +27,7 @@
         self.filesystemencoding = None
         self.debug = True
         self.track_resources = False
+        self.finalizing = False
         self.dlopenflags = rdynload._dlopen_default_mode()
 
     interpleveldefs = {
@@ -94,6 +95,8 @@
 
         'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
         'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
+
+        'is_finalizing'         : 'vm.is_finalizing',
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -668,6 +668,12 @@
         # be changed.
         assert sys.float_repr_style == "short"
 
+    def test_is_finalizing(self):
+        import sys
+        assert not sys.is_finalizing()
+        # xxx should also test when it is True, but maybe not worth the effort
+
+
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):
         import sys
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -308,3 +308,6 @@
         ec.w_coroutine_wrapper_fn = w_wrapper
     else:
         raise oefmt(space.w_TypeError, "callable expected, got %T", w_wrapper)
+
+def is_finalizing(space):
+    return space.wrap(space.sys.finalizing)

From pypy.commits at gmail.com  Thu Nov 10 10:00:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 10 Nov 2016 07:00:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use try-finally in ConnectionHandler to
 ensure resource cleanup and prevent timeout
Message-ID: <58248b7c.45ad1c0a.ae9d6.60b7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88292:9d22a6f232ff
Date: 2016-11-10 14:59 +0000
http://bitbucket.org/pypy/pypy/changeset/9d22a6f232ff/

Log:	Use try-finally in ConnectionHandler to ensure resource cleanup and
	prevent timeout

diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -1838,7 +1838,14 @@
                 else:
                     self.sock.close()
 
+            # PyPy change
             def run(self):
+                try:
+                    self._run()
+                finally:
+                    self.close()
+
+            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():

From pypy.commits at gmail.com  Thu Nov 10 10:03:27 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 07:03:27 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: cpyext
Message-ID: <58248c3f.87101c0a.e808b.6cec@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88293:111d8e1779ff
Date: 2016-11-09 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/111d8e1779ff/

Log:	cpyext

diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py
--- a/pypy/module/cpyext/bytearrayobject.py
+++ b/pypy/module/cpyext/bytearrayobject.py
@@ -83,7 +83,7 @@
             space.call_method(w_obj, 'extend', space.newbytes('\x00' * (newlen - oldlen)))
         elif oldlen > newlen:
             assert newlen >= 0
-            space.delslice(w_obj, space.wrap(newlen), space.wrap(oldlen))
+            space.delslice(w_obj, space.newint(newlen), space.newint(oldlen))
         return 0
     else:
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -118,14 +118,14 @@
 def PyString_FromStringAndSize(space, char_p, length):
     if char_p:
         s = rffi.charpsize2str(char_p, length)
-        return make_ref(space, space.wrap(s))
+        return make_ref(space, space.newtext(s))
     else:
         return rffi.cast(PyObject, new_empty_str(space, length))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyString_FromString(space, char_p):
     s = rffi.charp2str(char_p)
-    return space.wrap(s)
+    return space.newtext(s)
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
diff --git a/pypy/module/cpyext/mapping.py b/pypy/module/cpyext/mapping.py
--- a/pypy/module/cpyext/mapping.py
+++ b/pypy/module/cpyext/mapping.py
@@ -41,14 +41,14 @@
 def PyMapping_GetItemString(space, w_obj, key):
     """Return element of o corresponding to the object key or NULL on failure.
     This is the equivalent of the Python expression o[key]."""
-    w_key = space.wrap(rffi.charp2str(key))
+    w_key = space.newtext(rffi.charp2str(key))
     return space.getitem(w_obj, w_key)
 
 @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1)
 def PyMapping_SetItemString(space, w_obj, key, w_value):
     """Map the object key to the value v in object o. Returns -1 on failure.
     This is the equivalent of the Python statement o[key] = v."""
-    w_key = space.wrap(rffi.charp2str(key))
+    w_key = space.newtext(rffi.charp2str(key))
     space.setitem(w_obj, w_key, w_value)
     return 0
 
@@ -69,7 +69,7 @@
     This is equivalent to o[key], returning True on success and False
     on an exception.  This function always succeeds."""
     try:
-        w_key = space.wrap(rffi.charp2str(key))
+        w_key = space.newtext(rffi.charp2str(key))
         space.getitem(w_obj, w_key)
         return 1
     except:
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -60,7 +60,7 @@
     state.package_context = None, None
 
     if f_path is not None:
-        dict_w = {'__file__': space.wrap(f_path)}
+        dict_w = {'__file__': space.newtext(f_path)}
     else:
         dict_w = {}
     convert_method_defs(space, dict_w, methods, None, w_self, modname)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -151,7 +151,7 @@
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     if llfilename:
         filename = rffi.charp2str(llfilename)
-        w_filename = space.wrap(filename)
+        w_filename = space.newbytes(filename)
     else:
         w_filename = space.w_None
 
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -43,7 +43,7 @@
     filename, with a file mode given by mode, where mode has the same
     semantics as the standard C routine fopen().  On failure, return NULL."""
     w_filename = space.newbytes(rffi.charp2str(filename))
-    w_mode = space.wrap(rffi.charp2str(mode))
+    w_mode = space.newtext(rffi.charp2str(mode))
     return space.call_method(space.builtin, 'file', w_filename, w_mode)
 
 @cpython_api([PyObject], FILEP, error=lltype.nullptr(FILEP.TO))
@@ -81,7 +81,7 @@
         raise oefmt(space.w_NotImplementedError, 
             'PyFromFile(..., close) with close function not implemented')
     w_ret = space.allocate_instance(W_File, space.gettypefor(W_File))
-    w_ret.w_name = space.wrap(rffi.charp2str(name))
+    w_ret.w_name = space.newtext(rffi.charp2str(name))
     w_ret.check_mode_ok(rffi.charp2str(mode))
     w_ret.fp = fp
     return w_ret
@@ -96,7 +96,7 @@
 def PyFile_WriteString(space, s, w_p):
     """Write string s to file object p.  Return 0 on success or -1 on
     failure; the appropriate exception will be set."""
-    w_str = space.wrap(rffi.charp2str(s))
+    w_str = space.newtext(rffi.charp2str(s))
     space.call_method(w_p, "write", w_str)
     return 0
 
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -34,9 +34,9 @@
     if traceback.next is None:
         w_next_traceback = None
     else:
-        w_next_traceback = space.wrap(traceback.next)
+        w_next_traceback = traceback.next
     py_traceback.c_tb_next = rffi.cast(PyTracebackObject, make_ref(space, w_next_traceback))
-    py_traceback.c_tb_frame = rffi.cast(PyFrameObject, make_ref(space, space.wrap(traceback.frame)))
+    py_traceback.c_tb_frame = rffi.cast(PyFrameObject, make_ref(space, traceback.frame))
     rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti)
     rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno())
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -71,7 +71,7 @@
                                 tag="cpyext_1")
 
 def PyDescr_NewGetSet(space, getset, w_type):
-    return space.wrap(W_GetSetPropertyEx(getset, w_type))
+    return W_GetSetPropertyEx(getset, w_type)
 
 class W_MemberDescr(GetSetProperty):
     name = 'member_descriptor'
@@ -235,7 +235,7 @@
             if not name:
                 break
             name = rffi.charp2str(name)
-            w_descr = space.wrap(W_MemberDescr(member, w_type))
+            w_descr = W_MemberDescr(member, w_type)
             dict_w[name] = w_descr
             i += 1
 
@@ -326,7 +326,7 @@
             continue
         w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func,
                 wrapper_func_kwds, doc, func_voidp, offset=offset)
-        dict_w[method_name] = space.wrap(w_obj)
+        dict_w[method_name] = w_obj
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
 
@@ -477,7 +477,7 @@
               not (pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_slice)):
             self.flag_map_or_seq = 'M'
         if pto.c_tp_doc:
-            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
+            self.w_doc = space.newtext(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -719,7 +719,7 @@
             # point we might get into troubles by doing make_ref() when
             # things are not initialized yet.  So in this case, simply use
             # str2charp() and "leak" the string.
-        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        w_typename = space.getattr(w_type, space.newtext('__name__'))
         heaptype = rffi.cast(PyHeapTypeObject, pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
         from pypy.module.cpyext.bytesobject import PyString_AsString
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -285,7 +285,7 @@
     success, -1 in case of an error."""
     if not encoding:
         PyErr_BadArgument(space)
-    w_encoding = space.wrap(rffi.charp2str(encoding))
+    w_encoding = space.newtext(rffi.charp2str(encoding))
     setdefaultencoding(space, w_encoding)
     default_encoding[0] = '\x00'
     return 0
@@ -340,7 +340,7 @@
     is NULL."""
     if wchar_p:
         s = rffi.wcharpsize2unicode(wchar_p, length)
-        return make_ref(space, space.wrap(s))
+        return make_ref(space, space.newunicode(s))
     else:
         return rffi.cast(PyObject, new_empty_unicode(space, length))
 
@@ -373,7 +373,7 @@
         # This tracks CPython 2.7, in CPython 3.4 'utf-8' is hardcoded instead
         encoding = PyUnicode_GetDefaultEncoding(space)
     w_str = space.newbytes(rffi.charpsize2str(s, size))
-    w_encoding = space.wrap(rffi.charp2str(encoding))
+    w_encoding = space.newtext(rffi.charp2str(encoding))
     if errors:
         w_errors = space.newbytes(rffi.charp2str(errors))
     else:

From pypy.commits at gmail.com  Thu Nov 10 11:40:50 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 08:40:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: match CPython3's pbkdf2_hmac behaviour
 about the password and salt arguments
Message-ID: <5824a312.8ab81c0a.42c4a.870f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88295:38b208bbc61c
Date: 2016-11-10 17:40 +0100
http://bitbucket.org/pypy/pypy/changeset/38b208bbc61c/

Log:	match CPython3's pbkdf2_hmac behaviour about the password and salt
	arguments

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -187,7 +187,7 @@
 
 HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
 if HAS_FAST_PKCS5_PBKDF2_HMAC:
-    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+    @unwrap_spec(name=str, password='bytes', salt='bytes', rounds=int,
                  w_dklen=WrappedDefault(None))
     def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
         digest = ropenssl.EVP_get_digestbyname(name)
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -91,7 +91,9 @@
             from _hashlib import pbkdf2_hmac
         except ImportError:
             skip("Requires OpenSSL >= 1.1")
-        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        out = pbkdf2_hmac('sha1', b'password', b'salt', 1)
         assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
-        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        out = pbkdf2_hmac('sha1', b'password', b'salt', 2, None)
         assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
+        raises(TypeError, pbkdf2_hmac, 'sha1', 'password', b'salt', 1)
+        raises(TypeError, pbkdf2_hmac, 'sha1', b'password', 'salt', 1)

From pypy.commits at gmail.com  Thu Nov 10 11:40:48 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 08:40:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add a way to express "bytes" unwrapping
 via unwrap_spec
Message-ID: <5824a310.c1341c0a.942da.5233@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88294:8d2718505c72
Date: 2016-11-10 17:38 +0100
http://bitbucket.org/pypy/pypy/changeset/8d2718505c72/

Log:	add a way to express "bytes" unwrapping via unwrap_spec

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -159,6 +159,9 @@
     def visit_str0(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_bytes(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -296,6 +299,9 @@
     def visit_str0(self, typ):
         self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
 
+    def visit_bytes(self, typ):
+        self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
+
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
@@ -452,6 +458,9 @@
     def visit_str0(self, typ):
         self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
 
+    def visit_bytes(self, typ):
+        self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
+
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -256,6 +256,20 @@
         assert self.space.eq_w(space.call_function(w_app_g, space.wrap(True)),
                                space.wrap(True))
 
+    def test_interp2app_unwrap_spec_bytes(self):
+        # we can't use the "bytes" object for the unwrap_spec, because that's
+        # an alias for "str" on the underlying Python2
+        space = self.space
+        w = space.wrap
+        def g(space, b):
+            return space.newbytes(b)
+        app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'bytes'])
+        app_g2 = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'bytes'])
+        assert app_g is app_g2
+        w_app_g = space.wrap(app_g)
+        assert self.space.eq_w(space.call_function(w_app_g, space.newbytes("abc")),
+                               space.newbytes("abc"))
+
     def test_caching_methods(self):
         class Base(gateway.W_Root):
             def f(self):

From pypy.commits at gmail.com  Thu Nov 10 12:02:40 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 09:02:40 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix return value too
Message-ID: <5824a830.6249c20a.6f949.20fd@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88296:0a68df568b22
Date: 2016-11-10 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/0a68df568b22/

Log:	fix return value too

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -206,4 +206,4 @@
                 dklen, buf.raw)
             if not r:
                 raise ValueError
-            return space.wrap(buf.str(dklen))
+            return space.newbytes(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -92,6 +92,7 @@
         except ImportError:
             skip("Requires OpenSSL >= 1.1")
         out = pbkdf2_hmac('sha1', b'password', b'salt', 1)
+        assert type(out) is bytes
         assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
         out = pbkdf2_hmac('sha1', b'password', b'salt', 2, None)
         assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')

From pypy.commits at gmail.com  Thu Nov 10 12:04:16 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 09:04:16 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _hashlib. also refactor the
 NameFetcher to not need a space at all
Message-ID: <5824a890.11881c0a.a3572.9bd8@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88297:621430499498
Date: 2016-11-10 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/621430499498/

Log:	_hashlib. also refactor the NameFetcher to not need a space at all

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -24,32 +24,25 @@
     # algorithms.
     if rffi.cast(lltype.Signed, obj_name[0].c_alias):
         return
-    try:
-        space = global_name_fetcher.space
-        w_name = space.wrap(rffi.charp2str(obj_name[0].c_name))
-        global_name_fetcher.meth_names.append(w_name)
-    except OperationError as e:
-        global_name_fetcher.w_error = e
+    name = rffi.charp2str(obj_name[0].c_name)
+    global_name_fetcher.meth_names.append(name)
 
 class NameFetcher:
-    def setup(self, space):
-        self.space = space
+    def setup(self):
         self.meth_names = []
-        self.w_error = None
     def _cleanup_(self):
         self.__dict__.clear()
 global_name_fetcher = NameFetcher()
 
 def fetch_names(space):
-    global_name_fetcher.setup(space)
+    global_name_fetcher.setup()
     ropenssl.init_digests()
     ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH,
                              hash_name_mapper_callback, None)
-    if global_name_fetcher.w_error:
-        raise global_name_fetcher.w_error
     meth_names = global_name_fetcher.meth_names
     global_name_fetcher.meth_names = None
-    return space.call_function(space.w_frozenset, space.newlist(meth_names))
+    return space.call_function(space.w_frozenset, space.newlist(
+        [space.newtext(name) for name in meth_names]))
 
 class W_Hash(W_Root):
     NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)

From pypy.commits at gmail.com  Thu Nov 10 12:04:17 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 09:04:17 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _hashlib (also refactor
 NameFetcher to not need a space)
Message-ID: <5824a891.d5901c0a.da684.5b2f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88298:425adc283f7d
Date: 2016-11-10 18:03 +0100
http://bitbucket.org/pypy/pypy/changeset/425adc283f7d/

Log:	_hashlib (also refactor NameFetcher to not need a space)

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -88,7 +88,7 @@
 
     def descr_repr(self, space):
         addrstring = self.getaddrstring(space)
-        return space.wrap("<%s HASH object at 0x%s>" % (
+        return space.newtext("<%s HASH object at 0x%s>" % (
             self.name, addrstring))
 
     @unwrap_spec(string='bufferstr')
@@ -116,18 +116,18 @@
         for c in digest:
             result.append(hexdigits[(ord(c) >> 4) & 0xf])
             result.append(hexdigits[ ord(c)       & 0xf])
-        return space.wrap(result.build())
+        return space.newtext(result.build())
 
     def get_digest_size(self, space):
-        return space.wrap(self.digest_size)
+        return space.newint(self.digest_size)
 
     def get_block_size(self, space):
         digest_type = self.digest_type_by_name(space)
         block_size = ropenssl.EVP_MD_block_size(digest_type)
-        return space.wrap(block_size)
+        return space.newint(block_size)
 
     def get_name(self, space):
-        return space.wrap(self.name)
+        return space.newtext(self.name)
 
     def _digest(self, space):
         ctx = ropenssl.EVP_MD_CTX_new()
@@ -164,7 +164,7 @@
 def new(space, name, string=''):
     w_hash = W_Hash(space, name)
     w_hash.update(space, string)
-    return space.wrap(w_hash)
+    return w_hash
 
 # shortcut functions
 def make_new_hash(name, funcname):
@@ -200,4 +200,4 @@
                 dklen, buf.raw)
             if not r:
                 raise ValueError
-            return space.wrap(buf.str(dklen))
+            return space.newbytes(buf.str(dklen))

From pypy.commits at gmail.com  Thu Nov 10 12:09:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 10 Nov 2016 09:09:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix timeouts in test_urllib2_localnet.py
 and skip tests that now segfault
Message-ID: <5824a9e5.88711c0a.1ea76.1603@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88299:e7411922252b
Date: 2016-11-10 17:04 +0000
http://bitbucket.org/pypy/pypy/changeset/e7411922252b/

Log:	Fix timeouts in test_urllib2_localnet.py and skip tests that now
	segfault

diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -745,8 +745,7 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-
-            except (OSError, ValueError):
+            except:
                 self.close()
                 raise
 
diff --git a/lib-python/3/test/test_urllib2_localnet.py b/lib-python/3/test/test_urllib2_localnet.py
--- a/lib-python/3/test/test_urllib2_localnet.py
+++ b/lib-python/3/test/test_urllib2_localnet.py
@@ -544,6 +544,7 @@
         self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"])
 
     def test_https(self):
+        self.skipTest('Segfaults on PyPy')
         handler = self.start_https_server()
         context = ssl.create_default_context(cafile=CERT_localhost)
         data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context)
@@ -573,6 +574,7 @@
                          cadefault=True)
 
     def test_https_sni(self):
+        self.skipTest('Segfaults on PyPy')
         if ssl is None:
             self.skipTest("ssl module required")
         if not ssl.HAS_SNI:

From pypy.commits at gmail.com  Thu Nov 10 13:15:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 10:15:01 -0800 (PST)
Subject: [pypy-commit] pypy default: THe --pypy option in this test
 directory has no effect at all.
Message-ID: <5824b925.ca73c20a.522f4.3272@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88301:2ae7531dd919
Date: 2016-11-10 17:55 +0100
http://bitbucket.org/pypy/pypy/changeset/2ae7531dd919/

Log:	THe --pypy option in this test directory has no effect at all.

diff --git a/pypy/module/pypyjit/test/conftest.py b/pypy/module/pypyjit/test/conftest.py
--- a/pypy/module/pypyjit/test/conftest.py
+++ b/pypy/module/pypyjit/test/conftest.py
@@ -1,5 +1,7 @@
 def pytest_addoption(parser):
     group = parser.getgroup("pypyjit options")
     group.addoption("--pypy", action="store", default=None, dest="pypy_c",
-                    help="the location of the JIT enabled pypy-c")
-
+                    help="DEPRECATED: use this in test_pypy_c instead")
+# XXX kill the corresponding section in the buildbot run,
+# which (as far as I can tell) ignores that option entirely and does
+# the same as the regular py.test.

From pypy.commits at gmail.com  Thu Nov 10 13:14:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 10:14:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: unused imports
Message-ID: <5824b923.058a1c0a.3d288.77ba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88300:16df122f695c
Date: 2016-11-10 13:08 +0100
http://bitbucket.org/pypy/pypy/changeset/16df122f695c/

Log:	unused imports

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -280,8 +280,6 @@
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
 
-        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-        from pypy.objspace.std.bytesobject import W_BytesObject
         sub = self._op_val(space, w_sub, allow_char=True)
         if self._use_rstr_ops(space, w_sub):
             res = value.rfind(sub, start, end)

From pypy.commits at gmail.com  Thu Nov 10 13:15:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 10:15:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <5824b926.c6bdc20a.e3d52.3f61@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88302:c6e6087f1717
Date: 2016-11-10 19:10 +0100
http://bitbucket.org/pypy/pypy/changeset/c6e6087f1717/

Log:	merge heads

diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -745,8 +745,7 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-
-            except (OSError, ValueError):
+            except:
                 self.close()
                 raise
 
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -1838,7 +1838,14 @@
                 else:
                     self.sock.close()
 
+            # PyPy change
             def run(self):
+                try:
+                    self._run()
+                finally:
+                    self.close()
+
+            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
diff --git a/lib-python/3/test/test_urllib2_localnet.py b/lib-python/3/test/test_urllib2_localnet.py
--- a/lib-python/3/test/test_urllib2_localnet.py
+++ b/lib-python/3/test/test_urllib2_localnet.py
@@ -544,6 +544,7 @@
         self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"])
 
     def test_https(self):
+        self.skipTest('Segfaults on PyPy')
         handler = self.start_https_server()
         context = ssl.create_default_context(cafile=CERT_localhost)
         data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context)
@@ -573,6 +574,7 @@
                          cadefault=True)
 
     def test_https_sni(self):
+        self.skipTest('Segfaults on PyPy')
         if ssl is None:
             self.skipTest("ssl module required")
         if not ssl.HAS_SNI:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -159,6 +159,9 @@
     def visit_str0(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_bytes(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_fsencode(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
@@ -296,6 +299,9 @@
     def visit_str0(self, typ):
         self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
 
+    def visit_bytes(self, typ):
+        self.run_args.append("space.bytes_w(%s)" % (self.scopenext(),))
+
     def visit_fsencode(self, typ):
         self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),))
 
@@ -452,6 +458,9 @@
     def visit_str0(self, typ):
         self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
 
+    def visit_bytes(self, typ):
+        self.unwrap.append("space.bytes_w(%s)" % (self.nextarg(),))
+
     def visit_fsencode(self, typ):
         self.unwrap.append("space.fsencode_w(%s)" % (self.nextarg(),))
 
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -256,6 +256,20 @@
         assert self.space.eq_w(space.call_function(w_app_g, space.wrap(True)),
                                space.wrap(True))
 
+    def test_interp2app_unwrap_spec_bytes(self):
+        # we can't use the "bytes" object for the unwrap_spec, because that's
+        # an alias for "str" on the underlying Python2
+        space = self.space
+        w = space.wrap
+        def g(space, b):
+            return space.newbytes(b)
+        app_g = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'bytes'])
+        app_g2 = gateway.interp2app(g, unwrap_spec=[gateway.ObjSpace, 'bytes'])
+        assert app_g is app_g2
+        w_app_g = space.wrap(app_g)
+        assert self.space.eq_w(space.call_function(w_app_g, space.newbytes("abc")),
+                               space.newbytes("abc"))
+
     def test_caching_methods(self):
         class Base(gateway.W_Root):
             def f(self):
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -187,7 +187,7 @@
 
 HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
 if HAS_FAST_PKCS5_PBKDF2_HMAC:
-    @unwrap_spec(name=str, password=str, salt=str, rounds=int,
+    @unwrap_spec(name=str, password='bytes', salt='bytes', rounds=int,
                  w_dklen=WrappedDefault(None))
     def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
         digest = ropenssl.EVP_get_digestbyname(name)
@@ -206,4 +206,4 @@
                 dklen, buf.raw)
             if not r:
                 raise ValueError
-            return space.wrap(buf.str(dklen))
+            return space.newbytes(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -91,7 +91,10 @@
             from _hashlib import pbkdf2_hmac
         except ImportError:
             skip("Requires OpenSSL >= 1.1")
-        out = pbkdf2_hmac('sha1', 'password', 'salt', 1)
+        out = pbkdf2_hmac('sha1', b'password', b'salt', 1)
+        assert type(out) is bytes
         assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
-        out = pbkdf2_hmac('sha1', 'password', 'salt', 2, None)
+        out = pbkdf2_hmac('sha1', b'password', b'salt', 2, None)
         assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
+        raises(TypeError, pbkdf2_hmac, 'sha1', 'password', b'salt', 1)
+        raises(TypeError, pbkdf2_hmac, 'sha1', b'password', 'salt', 1)

From pypy.commits at gmail.com  Thu Nov 10 13:51:30 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 10 Nov 2016 10:51:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add a failing test about typecode y#
 (more failing tests in this file anyway)
Message-ID: <5824c1b2.6252c20a.52f52.46ef@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88303:e15a627f9333
Date: 2016-11-10 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/e15a627f9333/

Log:	add a failing test about typecode y# (more failing tests in this
	file anyway)

diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -200,3 +200,16 @@
             return PyLong_FromSsize_t(y);
             ''', PY_SSIZE_T_CLEAN=True)
         assert charbuf(b'12345') == 5
+
+    def test_pyarg_parse_with_py_ssize_t_bytes(self):
+        charbuf = self.import_parser(
+            '''
+            char *buf;
+            Py_ssize_t len = -1;
+            if (!PyArg_ParseTuple(args, "y#", &buf, &len)) {
+                return NULL;
+            }
+            return PyBytes_FromStringAndSize(buf, len);
+            ''', PY_SSIZE_T_CLEAN=True)
+        assert type(charbuf(b'12345')) is bytes
+        assert charbuf(b'12345') == b'12345'

From pypy.commits at gmail.com  Thu Nov 10 14:07:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 11:07:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Don't wrap a random interp-level
 exception inside an app-level
Message-ID: <5824c56b.46bb1c0a.33fda.714f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88304:81d063af711f
Date: 2016-11-10 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/81d063af711f/

Log:	Don't wrap a random interp-level exception inside an app-level
	SystemError when untranslated, because it makes testing harder.
	Unclear how to test the translated behavior now...

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -511,20 +511,6 @@
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
 
-def _convert_unexpected_exception_extra(space, e):
-    "NOT_RPYTHON"
-    if e.__class__.__name__ in (
-        'Skipped',     # list of exception class names that are ok
-        ):             # to get during ==untranslated tests== only
-        raise
-    # include the RPython-level traceback
-    exc = sys.exc_info()
-    import traceback, cStringIO
-    f = cStringIO.StringIO()
-    print >> f, "\nTraceback (interpreter-level):"
-    traceback.print_tb(exc[2], file=f)
-    return f.getvalue()
-
 @jit.dont_look_inside
 def get_converted_unexpected_exception(space, e):
     """This is used in two places when we get an non-OperationError
@@ -562,7 +548,9 @@
             debug_print_traceback()
             extra = '; internal traceback was dumped to stderr'
         else:
-            extra = _convert_unexpected_exception_extra(space, e)
+            # when untranslated, we don't wrap into an app-level
+            # SystemError (this makes debugging tests harder)
+            raise
         return OperationError(space.w_SystemError, space.wrap(
             "unexpected internal exception (please report a bug): %r%s" %
             (e, extra)))
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -773,6 +773,8 @@
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
     def test_system_error(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
         class UnexpectedException(Exception):
             pass
         space = self.space
@@ -788,6 +790,8 @@
                 'report a bug): UnexpectedException') in err
 
     def test_system_error_2(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
         class UnexpectedException(Exception):
             pass
         space = self.space

From pypy.commits at gmail.com  Thu Nov 10 14:37:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 10 Nov 2016 11:37:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Don't crash if SemLock.name is None
Message-ID: <5824cc5c.c3181c0a.91181.d5ea@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88305:6ae7eb63289e
Date: 2016-11-10 19:36 +0000
http://bitbucket.org/pypy/pypy/changeset/6ae7eb63289e/

Log:	Don't crash if SemLock.name is None

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -443,6 +443,8 @@
         self.name = name
 
     def name_get(self, space):
+        if self.name is None:
+            return space.w_None
         return space.newutf8(self.name)
 
     def kind_get(self, space):
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -41,6 +41,7 @@
         assert sem.kind == kind
         assert sem.maxvalue == maxvalue
         assert isinstance(sem.handle, int)
+        assert sem.name is None
 
         assert sem._count() == 0
         if sys.platform == 'darwin':

From pypy.commits at gmail.com  Thu Nov 10 15:16:10 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 10 Nov 2016 12:16:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: A failing test matching
 test_frame.py::FrameLocalsTest::test_clear_locals
Message-ID: <5824d58a.4b8f1c0a.88137.ab74@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88306:cbba0303793c
Date: 2016-11-10 20:15 +0000
http://bitbucket.org/pypy/pypy/changeset/cbba0303793c/

Log:	A failing test matching
	test_frame.py::FrameLocalsTest::test_clear_locals

diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -601,3 +601,29 @@
             pass
         sys.settrace(None)
         assert seen == ['call', 'exception', 'return']
+
+    def test_clear_locals(self):
+        def make_frames():
+            def outer():
+                x = 5
+                y = 6
+                def inner():
+                    z = x + 2
+                    1/0
+                    t = 9
+                return inner()
+            try:
+                outer()
+            except ZeroDivisionError as e:
+                tb = e.__traceback__
+                frames = []
+                while tb:
+                    frames.append(tb.tb_frame)
+                    tb = tb.tb_next
+            return frames
+
+        f, outer, inner = make_frames()
+        outer.clear()
+        inner.clear()
+        assert not outer.f_locals
+        assert not inner.f_locals

From pypy.commits at gmail.com  Fri Nov 11 01:36:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 22:36:54 -0800 (PST)
Subject: [pypy-commit] cffi default: Update the docs to the style r"""...C
 source..."""
Message-ID: <58256706.41d71c0a.24fa6.3db6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2809:2234672774e3
Date: 2016-11-11 07:36 +0100
http://bitbucket.org/cffi/cffi/changeset/2234672774e3/

Log:	Update the docs to the style r"""...C source..."""

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -58,7 +58,7 @@
     import cffi
 
     ffibuilder = cffi.FFI()
-    ffibuilder.set_source("package._foo", "real C code")   # <=
+    ffibuilder.set_source("package._foo", r"""real C code""")   # <=
     ffibuilder.cdef("C-like declarations with '...'")
 
     if __name__ == "__main__":
@@ -345,7 +345,9 @@
 module to generate.  In this mode, no C compiler is called.
 
 In **API mode,** the ``c_header_source`` argument is a string that
-will be pasted into the .c file generated.  This piece of C code
+will be pasted into the .c file generated.  Typically, it is specified as
+``r""" ...multiple lines of C code... """`` (the ``r`` prefix allows these
+lines to contain a literal ``\n``, for example).  This piece of C code
 typically contains some ``#include``, but may also contain more,
 like definitions for custom "wrapper" C functions.  The goal is that
 the .c file can be generated like this::
@@ -387,7 +389,7 @@
 
 .. code-block:: python
 
-    ffibuilder.set_source("mymodule", '''
+    ffibuilder.set_source("mymodule", r'''
     extern "C" {
         int somefunc(int somearg) { return real_cpp_func(somearg); }
     }
@@ -851,7 +853,7 @@
     import cffi
 
     ffi = cffi.FFI()
-    C_HEADER_SRC = '''
+    C_HEADER_SRC = r'''
         #include "somelib.h"
     '''
     C_KEYWORDS = dict(libraries=['somelib'])
diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst
--- a/doc/source/embedding.rst
+++ b/doc/source/embedding.rst
@@ -56,7 +56,7 @@
     with open('plugin.h') as f:
         ffibuilder.embedding_api(f.read())
 
-    ffibuilder.set_source("my_plugin", '''
+    ffibuilder.set_source("my_plugin", r'''
         #include "plugin.h"
     ''')
 
@@ -211,7 +211,9 @@
   As in the example above, you can also use the same ``foo.h`` from
   ``ffibuilder.set_source()``::
 
-      ffibuilder.set_source('module_name', '#include "foo.h"')
+      ffibuilder.set_source('module_name', r'''
+          #include "foo.h"
+      ''')
 
 
 .. __: using.html#working
@@ -404,7 +406,7 @@
         extern "Python" int mycb(int);
     """)
 
-    ffibuilder.set_source("my_plugin", """
+    ffibuilder.set_source("my_plugin", r"""
 
         static int mycb(int);   /* the callback: forward declaration, to make
                                    it accessible from the C code that follows */
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -72,7 +72,7 @@
     ffibuilder = FFI()
 
     ffibuilder.set_source("_example",
-        """ // passed to the real C compiler
+       r""" // passed to the real C compiler
             #include <sys/types.h>
             #include <pwd.h>
         """,
@@ -195,7 +195,7 @@
     ffibuilder.cdef("int foo(int *, int *, int);")
 
     ffibuilder.set_source("_example",
-    """
+    r"""
         static int foo(int *buffer_in, int *buffer_out, int x)
         {
             /* some algorithm that is seriously faster in C than in Python */
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -479,7 +479,7 @@
 
         void library_function(int(*callback)(int, int));
     """)
-    ffibuilder.set_source("_my_example", """
+    ffibuilder.set_source("_my_example", r"""
         #include <some_library.h>
     """)
 
@@ -546,7 +546,7 @@
 
         extern "Python" void my_event_callback(event_t *, void *);
     """)
-    ffibuilder.set_source("_demo_cffi", """
+    ffibuilder.set_source("_demo_cffi", r"""
         #include <the_event_library.h>
     """)
 
@@ -613,7 +613,7 @@
 
 ::
 
-    ffibuilder.set_source("_demo_cffi", """
+    ffibuilder.set_source("_demo_cffi", r"""
         #include <the_event_library.h>
 
         static void my_event_callback(widget_t *, event_t *);
@@ -629,7 +629,7 @@
         extern "Python" int f(int);
         int my_algo(int);
     """)
-    ffibuilder.set_source("_example_cffi", """
+    ffibuilder.set_source("_example_cffi", r"""
         static int f(int);   /* the forward declaration */
 
         static int my_algo(int n) {
@@ -673,7 +673,7 @@
     ffibuilder.cdef("""
         extern "Python+C" int f(int);      /* not static */
     """)
-    ffibuilder.set_source("_example_cffi", """
+    ffibuilder.set_source("_example_cffi", r"""
         /* the forward declaration, setting a gcc attribute
            (this line could also be in some .h file, to be included
            both here and in the other C files of the project) */
@@ -834,7 +834,7 @@
         int (*python_callback)(int how_many, int *values);
         void *const c_callback;   /* pass this const ptr to C routines */
     """)
-    ffibuilder.set_source("_example", """
+    ffibuilder.set_source("_example", r"""
         #include <stdarg.h>
         #include <alloca.h>
         static int (*python_callback)(int how_many, int *values);

From pypy.commits at gmail.com  Fri Nov 11 02:05:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 23:05:19 -0800 (PST)
Subject: [pypy-commit] pypy default: follow-up on 81d063af711f: only run
 this test in "pypy py.test -A"
Message-ID: <58256daf.4251c20a.ae8bd.0690@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88307:ba5310864c14
Date: 2016-11-11 08:04 +0100
http://bitbucket.org/pypy/pypy/changeset/ba5310864c14/

Log:	follow-up on 81d063af711f: only run this test in "pypy py.test -A"

diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -92,7 +92,6 @@
         from __pypy__ import do_what_I_mean
         x = do_what_I_mean()
         assert x == 42
-        raises(SystemError, do_what_I_mean, 1)
 
     def test_list_strategy(self):
         from __pypy__ import strategy
@@ -136,9 +135,19 @@
 class AppTestJitFeatures(object):
     spaceconfig = {"translation.jit": True}
 
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
     def test_jit_backend_features(self):
         from __pypy__ import jit_backend_features
         supported_types = jit_backend_features
         assert isinstance(supported_types, list)
         for x in supported_types:
             assert x in ['floats', 'singlefloats', 'longlong']
+
+    def test_do_what_I_mean_error(self):
+        if not self.runappdirect:
+            skip("we don't wrap a random exception inside SystemError "
+                 "when untranslated, because it makes testing harder")
+        from __pypy__ import do_what_I_mean
+        raises(SystemError, do_what_I_mean, 1)

From pypy.commits at gmail.com  Fri Nov 11 02:13:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 23:13:02 -0800 (PST)
Subject: [pypy-commit] cffi default: Backed out changeset 0ecb431b2b94
Message-ID: <58256f7e.41d71c0a.24fa6.495e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2810:f5ef766cce5e
Date: 2016-11-11 08:12 +0100
http://bitbucket.org/cffi/cffi/changeset/f5ef766cce5e/

Log:	Backed out changeset 0ecb431b2b94

	I think that the init function is not always exported any more now,
	which breaks everything.

diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('void')
+        prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:

From pypy.commits at gmail.com  Fri Nov 11 02:19:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 23:19:53 -0800 (PST)
Subject: [pypy-commit] cffi default: A different fix for 0ecb431b2b94
Message-ID: <58257119.cbd51c0a.13da4.5083@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2811:31a922c41068
Date: 2016-11-11 08:19 +0100
http://bitbucket.org/cffi/cffi/changeset/31a922c41068/

Log:	A different fix for 0ecb431b2b94

diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 

From pypy.commits at gmail.com  Fri Nov 11 02:22:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 10 Nov 2016 23:22:26 -0800 (PST)
Subject: [pypy-commit] pypy default: import cffi/0ecb431b2b94
Message-ID: <582571b2.c89cc20a.57d54.8c06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88308:c6501a0ed0d6
Date: 2016-11-11 08:20 +0100
http://bitbucket.org/pypy/pypy/changeset/c6501a0ed0d6/

Log:	import cffi/0ecb431b2b94

diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('void')
+        prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:

From pypy.commits at gmail.com  Fri Nov 11 02:59:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 10 Nov 2016 23:59:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.6.0 for
 changeset aff251e54385
Message-ID: <58257a50.12111c0a.f0f31.584a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88309:2e4a0d78a7ae
Date: 2016-11-11 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/2e4a0d78a7ae/

Log:	Added tag release-pypy2.7-v5.6.0 for changeset aff251e54385

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0

From pypy.commits at gmail.com  Fri Nov 11 02:59:14 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 10 Nov 2016 23:59:14 -0800 (PST)
Subject: [pypy-commit] pypy default: tweak repackage script for (yet
 another) different branch name
Message-ID: <58257a52.a285c20a.6e708.0fdc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88310:94adb652ea1f
Date: 2016-11-11 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/94adb652ea1f/

Log:	tweak repackage script for (yet another) different branch name

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -2,7 +2,7 @@
 maj=5
 min=6
 rev=0
-branchname=release-$maj.x  # ==OR== release-$maj.$min.x
+branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
@@ -39,7 +39,7 @@
 zip -rq $rel-$plat.zip $rel-$plat
 rm -rf $rel-$plat
 
-# Do this after creating a tag, note the untarred directory is pypy-pypy-<hash>
+# Requires a valid $tagname, note the untarred directory is pypy-pypy-<hash>
 # so make sure there is not another one
 wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
 tar -xf $tagname.tar.bz2

From pypy.commits at gmail.com  Fri Nov 11 03:20:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 11 Nov 2016 00:20:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Broke some pre-translated case (r_int64
 + r_int32). Fix and tests
Message-ID: <58257f31.12111c0a.f0f31.5f8f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88311:1be7f6327a03
Date: 2016-11-11 09:19 +0100
http://bitbucket.org/pypy/pypy/changeset/1be7f6327a03/

Log:	Broke some pre-translated case (r_int64 + r_int32). Fix and tests

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -323,96 +323,165 @@
 
     def __add__(self, other):
         x = long(self)
-        y = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return x + other
+        y = long(other)
         return self._widen(other, x + y)
-    __radd__ = __add__
+
+    def __radd__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other + x
+        y = long(other)
+        return self._widen(other, x + y)
 
     def __sub__(self, other):
         x = long(self)
-        y = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return x - other
+        y = long(other)
         return self._widen(other, x - y)
 
     def __rsub__(self, other):
         y = long(self)
-        x = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return other - y
+        x = long(other)
         return self._widen(other, x - y)
 
     def __mul__(self, other):
         x = long(self)
-        y = other      # may be a float
-        z = x * y
-        if isinstance(z, (int, long)):
-            z = self._widen(other, z)
-        return z
-    __rmul__ = __mul__
+        if not isinstance(other, (int, long)):
+            return x * other
+        y = long(other)
+        return self._widen(other, x * y)
+
+    def __rmul__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other * x
+        y = long(other)
+        return self._widen(other, x * y)
 
     def __div__(self, other):
         x = long(self)
-        y = other      # may be a float
-        return self._widen(other, x / y)
-
-    __floordiv__ = __div__
+        if not isinstance(other, (int, long)):
+            return x / other
+        y = long(other)
+        return self._widen(other, x // y)
 
     def __rdiv__(self, other):
         y = long(self)
-        x = other      # may be a float
-        return self._widen(other, x / y)
+        if not isinstance(other, (int, long)):
+            return other / y
+        x = long(other)
+        return self._widen(other, x // y)
 
-    __rfloordiv__ = __rdiv__
+    def __floordiv__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return x // other
+        y = long(other)
+        return self._widen(other, x // y)
+
+    def __rfloordiv__(self, other):
+        y = long(self)
+        if not isinstance(other, (int, long)):
+            return other // y
+        x = long(other)
+        return self._widen(other, x // y)
 
     def __mod__(self, other):
         x = long(self)
-        y = other      # not rpython if it is a float
+        if not isinstance(other, (int, long)):
+            return x % other
+        y = long(other)
         return self._widen(other, x % y)
 
     def __rmod__(self, other):
         y = long(self)
-        x = other      # not rpython if it is a float
+        if not isinstance(other, (int, long)):
+            return other % y
+        x = long(other)
         return self._widen(other, x % y)
 
     def __divmod__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return divmod(x, other)
         y = long(other)
         res = divmod(x, y)
-        return (self.__class__(res[0]), self.__class__(res[1]))
+        return (self._widen(other, res[0]), self._widen(other, res[1]))
 
     def __lshift__(self, n):
         x = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         y = long(n)
         return self.__class__(x << y)
 
     def __rlshift__(self, n):
         y = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         x = long(n)
-        return self._widen(n, x << y)
+        return n.__class__(x << y)
 
     def __rshift__(self, n):
         x = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         y = long(n)
-        return self._widen(n, x >> y)
+        return self.__class__(x >> y)
 
     def __rrshift__(self, n):
         y = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         x = long(n)
-        return self._widen(n, x >> y)
+        return n.__class__(x >> y)
 
     def __or__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x | other
         y = long(other)
         return self._widen(other, x | y)
-    __ror__ = __or__
+
+    def __ror__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other | x
+        y = long(other)
+        return self._widen(other, x | y)
 
     def __and__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x & other
         y = long(other)
         return self._widen(other, x & y)
-    __rand__ = __and__
+
+    def __rand__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other & x
+        y = long(other)
+        return self._widen(other, x & y)
 
     def __xor__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x ^ other
         y = long(other)
         return self._widen(other, x ^ y)
-    __rxor__ = __xor__
+
+    def __rxor__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other ^ x
+        y = long(other)
+        return self._widen(other, x ^ y)
 
     def __neg__(self):
         x = long(self)
@@ -431,12 +500,16 @@
 
     def __pow__(self, other, m=None):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return pow(x, other, m)
         y = long(other)
         res = pow(x, y, m)
         return self._widen(other, res)
 
     def __rpow__(self, other, m=None):
         y = long(self)
+        if not isinstance(other, (int, long)):
+            return pow(other, y, m)
         x = long(other)
         res = pow(x, y, m)
         return self._widen(other, res)
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -603,11 +603,75 @@
         assert r_uint64(self._64_umax) + r_uint64(1) == r_uint64(0)
         assert r_uint64(0) - r_uint64(1) == r_uint64(self._64_umax)
 
-    def test_operation_with_float(self):
-        def f(x):
-            assert r_longlong(x) + 0.5 == 43.5
-            assert r_longlong(x) - 0.5 == 42.5
-            assert r_longlong(x) * 0.5 == 21.5
-            assert r_longlong(x) / 0.8 == 53.75
-        f(43)
-        interpret(f, [43])
+
+def test_operation_with_float():
+    def f(x):
+        assert r_longlong(x) + 0.5 == 43.5
+        assert r_longlong(x) - 0.5 == 42.5
+        assert r_longlong(x) * 0.5 == 21.5
+        assert r_longlong(x) / 0.8 == 53.75
+    f(43)
+    interpret(f, [43])
+
+def test_int64_plus_int32():
+    assert r_uint64(1234567891234) + r_uint32(1) == r_uint64(1234567891235)
+
+def test_fallback_paths():
+
+    def make(pattern):
+        def method(self, other, *extra):
+            if extra:
+                assert extra == (None,)   # for 'pow'
+            if type(other) is long:
+                return pattern % other
+            else:
+                return NotImplemented
+        return method
+
+    class A(object):
+        __add__  = make("a+%d")
+        __radd__ = make("%d+a")
+        __sub__  = make("a-%d")
+        __rsub__ = make("%d-a")
+        __mul__  = make("a*%d")
+        __rmul__ = make("%d*a")
+        __div__  = make("a/%d")
+        __rdiv__ = make("%d/a")
+        __floordiv__  = make("a//%d")
+        __rfloordiv__ = make("%d//a")
+        __mod__  = make("a%%%d")
+        __rmod__ = make("%d%%a")
+        __and__  = make("a&%d")
+        __rand__ = make("%d&a")
+        __or__   = make("a|%d")
+        __ror__  = make("%d|a")
+        __xor__  = make("a^%d")
+        __rxor__ = make("%d^a")
+        __pow__  = make("a**%d")
+        __rpow__ = make("%d**a")
+
+    a = A()
+    assert r_uint32(42) + a == "42+a"
+    assert a + r_uint32(42) == "a+42"
+    assert r_uint32(42) - a == "42-a"
+    assert a - r_uint32(42) == "a-42"
+    assert r_uint32(42) * a == "42*a"
+    assert a * r_uint32(42) == "a*42"
+    assert r_uint32(42) / a == "42/a"
+    assert a / r_uint32(42) == "a/42"
+    assert r_uint32(42) // a == "42//a"
+    assert a // r_uint32(42) == "a//42"
+    assert r_uint32(42) % a == "42%a"
+    assert a % r_uint32(42) == "a%42"
+    py.test.raises(TypeError, "a << r_uint32(42)")
+    py.test.raises(TypeError, "r_uint32(42) << a")
+    py.test.raises(TypeError, "a >> r_uint32(42)")
+    py.test.raises(TypeError, "r_uint32(42) >> a")
+    assert r_uint32(42) & a == "42&a"
+    assert a & r_uint32(42) == "a&42"
+    assert r_uint32(42) | a == "42|a"
+    assert a | r_uint32(42) == "a|42"
+    assert r_uint32(42) ^ a == "42^a"
+    assert a ^ r_uint32(42) == "a^42"
+    assert r_uint32(42) ** a == "42**a"
+    assert a ** r_uint32(42) == "a**42"

From pypy.commits at gmail.com  Fri Nov 11 07:47:35 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 04:47:35 -0800 (PST)
Subject: [pypy-commit] pypy default: since I keep getting the question,
 add an FAQ entry about Windows 64 support
Message-ID: <5825bde7.4a0e1c0a.2bb0.fbd2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88312:b6a48cd3fd8c
Date: 2016-11-11 13:46 +0100
http://bitbucket.org/pypy/pypy/changeset/b6a48cd3fd8c/

Log:	since I keep getting the question, add an FAQ entry about Windows 64
	support

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -407,3 +407,17 @@
 long and gnarly. On the other hand, all our infrastructure (buildbots,
 benchmarking, etc) would have to be adapted. So unless somebody steps up and
 volunteers to do all that work, it will likely not happen.
+
+
+What is needed for Windows 64 support of PyPy?
+-----------------------------------------------
+
+First, please note that the Windows 32 PyPy binary works just fine on Windows
+64. The only problem is that it only supports up to 4GB of heap per process.
+
+As to real Windows 64 support: Currently we don't have an active PyPy developer
+whose main development platform is Windows. So if you are interested in getting
+Windows 64 support, we encourage you to volunteer to make it happen! Another
+option would be to pay some PyPy developers to implement Windows 64 support,
+but so far there doesn't seem to be an overwhelming commercial interest in it.
+

From pypy.commits at gmail.com  Fri Nov 11 07:49:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 11 Nov 2016 04:49:56 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: prepare the download page (not
 regenerated yet)
Message-ID: <5825be74.d39a1c0a.435b2.8fb8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r815:66eb03408b59
Date: 2016-11-11 14:49 +0200
http://bitbucket.org/pypy/pypy.org/changeset/66eb03408b59/

Log:	prepare the download page (not regenerated yet)

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -459,20 +459,18 @@
 
 Here are the checksums for each of the downloads
 
-pypy2.7-v5.4.1 md5::
+pypy2.7-v5.6.0 md5::
 
-    425ffedf0db4dd737d450aa064ae0e7a  pypy2-v5.4.1-linux-armel.tar.bz2
-    15f41409cbadbde3ef22ee60ded8579a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    5940ea0c1077e3bc2ba461bf55800abb  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    23cbad540abff48ea67f5b75368344be  pypy2-v5.4.1-linux32.tar.bz2
-    2f4a82ae306f3d30d486fb945ca9c6be  pypy2-v5.4.1-linux64.tar.bz2
-    a18cbd990c97f75a08235a35e546a637  pypy2-v5.4.1-osx64.tar.bz2
-    90b0650dda1b5bf33f6119f20ba9c3b6  pypy2-v5.4.1-ppc64.tar.bz2
-    9a2c1b3124151b79e68f36eb2d4891d1  pypy2-v5.4.1-ppc64le.tar.bz2
-    826d4b48e43c84a50dc7e2adc2cb69d4  pypy2-v5.4.1-s390x.tar.bz2
-    d1d197d16331aa23a7fd4b5d4c3c1717  pypy2-v5.4.1-src.tar.bz2
-    1aab9fe6e7c03e959cde466819034bab  pypy2-v5.4.1-src.zip
-    b04aad943aac92862a73b1fd90157a00  pypy2-v5.4.1-win32.zip
+    f804f1ee631125aa0a3c8eb8a9f09f20  pypy2-v5.6.0-linux32.tar.bz2
+    4d7514f43c5e92e2d066480db86d1932  pypy2-v5.6.0-linux64.tar.bz2
+    112b340c407f406dbf79ce715f111467  pypy2-v5.6.0-linux-armel.tar.bz2
+    736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+    65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+    2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
+    f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
+    c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
+    d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
+    3579eeea2caa74accf5d35a8cb406a83  pypy2-v5.6.0-win32.zip
 
 pypy3.3-v5.5.0-alpha md5::
 
@@ -490,20 +488,18 @@
     2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
     009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
-pypy2.7-5.4.1 sha1::
+pypy2.7-5.6.0 sha1::
 
-    a54b2a8b6def85663b10fd956d51fbd052954b83  pypy2-v5.4.1-linux-armel.tar.bz2
-    61c9a5269d6d414c4a1d7c41bbc6a45da318f138  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    ea48ffe40887e25adcf1969a6b0e25dbe42a2457  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    bf0ac4668323abead04dd0948a2e541a158e4a00  pypy2-v5.4.1-linux32.tar.bz2
-    d16601b04987381c4922a19dacb0ca2591a3b566  pypy2-v5.4.1-linux64.tar.bz2
-    4084b7db8ee16a52e7ecb89d10765d961864c67c  pypy2-v5.4.1-osx64.tar.bz2
-    98eac6f32f4fe8fce6eb0f0594163f3e01dccced  pypy2-v5.4.1-ppc64.tar.bz2
-    fc676af00b356ae48602b531675774f2badfc4fd  pypy2-v5.4.1-ppc64le.tar.bz2
-    70b736f1fdb92ae9916630b7cc8954ed92490f64  pypy2-v5.4.1-s390x.tar.bz2
-    1143948f50b1b95b55465b246639b4c48251e38e  pypy2-v5.4.1-src.tar.bz2
-    a6a9ce0defb401d8777d8af5949d23393416a390  pypy2-v5.4.1-src.zip
-    49c2ad75f1531730f7ee466d833d318333915ce0  pypy2-v5.4.1-win32.zip
+    78124bd560ab5bd4902a4f55e776c917d851736c  pypy2-v5.6.0-linux32.tar.bz2
+    6a5a254ceeb92b0bb9c2e3bfdcaf5a0d9f34df83  pypy2-v5.6.0-linux64.tar.bz2
+    8b452ae867f0a5131427386263abe458bf8db845  pypy2-v5.6.0-linux-armel.tar.bz2
+    190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+    0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+    5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
+    35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
+    ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
+    bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
+    a7d5d5287c84df3cbbebadee3c455cd4f88528d6  pypy2-v5.6.0-win32.zip
 
 pypy3.3-v5.5.0-alpha sha1::
 
@@ -516,20 +512,18 @@
     9eb6aad2d41f7db16c19ae6e438c8f195b0d44fc  pypy3.3-v5.5.0-alpha-src.tar.bz2
     b670cdc685c323b11bfdca2a15f12c5953e6d8b4  pypy3.3-v5.5.0-alpha-src.zip
 
-pypy2.7-5.4.1 sha256::
+pypy2.7-5.6.0 sha256::
 
-    a1eb5f672aae62606176305e52a51b060ba974b6181ebefcd2c555ecf5f8614f  pypy2-v5.4.1-linux-armel.tar.bz2
-    2c4befc4517adec874155a8b6fa0b9d18388943d4ffe778002072db7783e417a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-    b38646519ee1a888c68f8f4713c122867b4b36693c8acabb38eb827a9d2d51f9  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-    6d1e2386ec1e05dffed493aa2d5e6db5cf5de18d7350d44b85f2e45aa5c9a774  pypy2-v5.4.1-linux32.tar.bz2
-    9c85319778224d7fb0c348f55fe3fada15bb579c5f3870a13ad63b42a737dd72  pypy2-v5.4.1-linux64.tar.bz2
-    ae9329c8f0a6df431c6224c27c634f998688ac803e8d100cee9a774e6bba38b5  pypy2-v5.4.1-osx64.tar.bz2
-    ff118f2dc2393c821595fa7872e4d7bdae2a9a16a854759e103608df383d765a  pypy2-v5.4.1-ppc64.tar.bz2
-    684e862cdb281a22c95d73012f7d2693be9fd9cdd171784256da1514ae1c8164  pypy2-v5.4.1-ppc64le.tar.bz2
-    4004c86bf2dfdd1d92120f514d296af7602e0c5e2c6526a14ff8b5508c0fe8f7  pypy2-v5.4.1-s390x.tar.bz2
-    45dbc50c81498f6f1067201b8fc887074b43b84ee32cc47f15e7db17571e9352  pypy2-v5.4.1-src.tar.bz2
-    54b23c11a92dd6328a58787c3d719189f0aa24c872b479acf50094dfb4be0a46  pypy2-v5.4.1-src.zip
-    ec729218a820bc2aa2cf1fcacf9de0fee9e04144fe138596198a6b4615505e03  pypy2-v5.4.1-win32.zip
+    5d4ad43aed5c5e147f7e7c84766c729f34f63b714b6d224e912a2bb42dc95d62  pypy2-v5.6.0-linux32.tar.bz2
+    aad55328cb0673a60b2633dcc3c36cf452917ac906b577eb3aed5876a7666fca  pypy2-v5.6.0-linux64.tar.bz2
+    2d1c7820f6368c92bb43a153d2c995f70aa183ff8f1df6916b0d2e57838d8a30  pypy2-v5.6.0-linux-armel.tar.bz2
+    2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+    0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+    6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
+    534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
+    7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
+    f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip
+    bab4fa37ef2d32660e291393d955a4e951d5e883abea8bee83be1ec044ddcaac  pypy2-v5.6.0-win32.zip
 
 pypy3.3-v5.5.0-alpha sha256::
 

From pypy.commits at gmail.com  Fri Nov 11 07:57:54 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 04:57:54 -0800 (PST)
Subject: [pypy-commit] pypy default: link to technical discussion
Message-ID: <5825c052.8f95c20a.34d91.8e93@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88313:1e642bc089e1
Date: 2016-11-11 13:57 +0100
http://bitbucket.org/pypy/pypy/changeset/1e642bc089e1/

Log:	link to technical discussion

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -417,7 +417,8 @@
 
 As to real Windows 64 support: Currently we don't have an active PyPy developer
 whose main development platform is Windows. So if you are interested in getting
-Windows 64 support, we encourage you to volunteer to make it happen! Another
+Windows 64 support, we encourage you to volunteer `to make it happen`_! Another
 option would be to pay some PyPy developers to implement Windows 64 support,
 but so far there doesn't seem to be an overwhelming commercial interest in it.
 
+.. _`to make it happen`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation

From pypy.commits at gmail.com  Fri Nov 11 08:04:43 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 11 Nov 2016 05:04:43 -0800 (PST)
Subject: [pypy-commit] pypy default: make link relative
Message-ID: <5825c1eb.031f1c0a.b0c7f.0530@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88314:bde69606228c
Date: 2016-11-11 15:04 +0200
http://bitbucket.org/pypy/pypy/changeset/bde69606228c/

Log:	make link relative

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -421,4 +421,4 @@
 option would be to pay some PyPy developers to implement Windows 64 support,
 but so far there doesn't seem to be an overwhelming commercial interest in it.
 
-.. _`to make it happen`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+.. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation

From pypy.commits at gmail.com  Fri Nov 11 11:42:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 11 Nov 2016 08:42:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add 2 failing tests for views of
 memoryviews
Message-ID: <5825f4ed.cf3fc20a.1e77d.e6b9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88315:84366130bc20
Date: 2016-11-11 16:41 +0000
http://bitbucket.org/pypy/pypy/changeset/84366130bc20/

Log:	Add 2 failing tests for views of memoryviews

diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -239,6 +239,21 @@
         m3[1:5:2] = memoryview(b"xyXY").cast('h')
         assert data == bytearray(eval("b'abxyefXYij'"))
 
+    def test_cast_and_slice(self):
+        import array
+        data = array.array('h', [1, 2])
+        m = memoryview(memoryview(data).cast('B'))
+        assert len(m[2:4:1]) == 2
+
+    def test_cast_and_view(self):
+        import array
+        data = array.array('h', [1, 2])
+        m1 = memoryview(data).cast('B')
+        m2 = memoryview(m1)
+        assert m2.strides == m1.strides
+        assert m2.itemsize == m1.itemsize
+        assert m2.shape == m1.shape
+
 class MockBuffer(Buffer):
     def __init__(self, space, w_arr, w_dim, w_fmt, \
                  w_itemsize, w_strides, w_shape):

From pypy.commits at gmail.com  Fri Nov 11 11:44:20 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:20 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _minimal_curses
Message-ID: <5825f564.d4301c0a.163d9.5d19@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88316:50b6102e436c
Date: 2016-11-10 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/50b6102e436c/

Log:	_minimal_curses

diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py
--- a/pypy/module/_minimal_curses/interp_curses.py
+++ b/pypy/module/_minimal_curses/interp_curses.py
@@ -22,8 +22,8 @@
 def convert_error(space, error):
     msg = error.msg
     w_module = space.getbuiltinmodule('_minimal_curses')
-    w_exception_class = space.getattr(w_module, space.wrap('error'))
-    w_exception = space.call_function(w_exception_class, space.wrap(msg))
+    w_exception_class = space.getattr(w_module, space.newtext('error'))
+    w_exception = space.call_function(w_exception_class, space.newtext(msg))
     return OperationError(w_exception_class, w_exception)
 
 def _curses_setupterm_null(fd):
@@ -44,9 +44,9 @@
 def setupterm(space, w_termname=None, fd=-1):
     if fd == -1:
         w_stdout = space.getattr(space.getbuiltinmodule('sys'),
-                                 space.wrap('stdout'))
+                                 space.newtext('stdout'))
         fd = space.int_w(space.call_function(space.getattr(w_stdout,
-                                             space.wrap('fileno'))))
+                                             space.newtext('fileno'))))
     try:
         if space.is_none(w_termname):
             _curses_setupterm_null(fd)

From pypy.commits at gmail.com  Fri Nov 11 11:44:22 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:22 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: pypyjit
Message-ID: <5825f566.542e1c0a.32a09.ba84@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88317:f4971678b89f
Date: 2016-11-10 18:17 +0100
http://bitbucket.org/pypy/pypy/changeset/f4971678b89f/

Log:	pypyjit

diff --git a/pypy/module/pypyjit/hooks.py b/pypy/module/pypyjit/hooks.py
--- a/pypy/module/pypyjit/hooks.py
+++ b/pypy/module/pypyjit/hooks.py
@@ -18,9 +18,9 @@
             try:
                 try:
                     space.call_function(cache.w_abort_hook,
-                        space.wrap(jitdriver.name),
+                        space.newtext(jitdriver.name),
                         wrap_greenkey(space, jitdriver, greenkey, greenkey_repr),
-                        space.wrap(Counters.counter_names[reason]),
+                        space.newtext(Counters.counter_names[reason]),
                         space.newlist(oplist_w)
                     )
                 except OperationError as e:
@@ -38,7 +38,7 @@
             try:
                 try:
                     space.call_function(cache.w_trace_too_long_hook,
-                        space.wrap(jitdriver.name),
+                        space.newtext(jitdriver.name),
                         wrap_greenkey(space, jitdriver, greenkey, greenkey_repr))
                 except OperationError as e:
                     e.write_unraisable(space, "jit hook", cache.w_trace_too_long_hook)
@@ -69,7 +69,7 @@
             try:
                 try:
                     space.call_function(cache.w_compile_hook,
-                                        space.wrap(w_debug_info))
+                                        w_debug_info)
                 except OperationError as e:
                     e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
             finally:
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -361,19 +361,19 @@
     w_times = space.newdict()
     if ll_times:
         for i in range(len(ll_times)):
-            w_key = space.newtuple([space.wrap(ll_times[i].type),
-                                    space.wrap(ll_times[i].number)])
+            w_key = space.newtuple([space.newtext(ll_times[i].type),
+                                    space.newint(ll_times[i].number)])
             space.setitem(w_times, w_key,
-                          space.wrap(ll_times[i].counter))
+                          space.newint(ll_times[i].counter))
     w_counters = space.newdict()
     for i, counter_name in enumerate(Counters.counter_names):
         v = jit_hooks.stats_get_counter_value(None, i)
-        space.setitem_str(w_counters, counter_name, space.wrap(v))
+        space.setitem_str(w_counters, counter_name, space.newint(v))
     w_counter_times = space.newdict()
     tr_time = jit_hooks.stats_get_times_value(None, Counters.TRACING)
-    space.setitem_str(w_counter_times, 'TRACING', space.wrap(tr_time))
+    space.setitem_str(w_counter_times, 'TRACING', space.newfloat(tr_time))
     b_time = jit_hooks.stats_get_times_value(None, Counters.BACKEND)
-    space.setitem_str(w_counter_times, 'BACKEND', space.wrap(b_time))
+    space.setitem_str(w_counter_times, 'BACKEND', space.newfloat(b_time))
     return W_JitInfoSnapshot(space, w_times, w_counters, w_counter_times)
 
 def get_stats_asmmemmgr(space):

From pypy.commits at gmail.com  Fri Nov 11 11:44:24 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:24 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: various
Message-ID: <5825f568.8c1f1c0a.60aeb.ffb4@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88318:d1caa90c2615
Date: 2016-11-10 18:19 +0100
http://bitbucket.org/pypy/pypy/changeset/d1caa90c2615/

Log:	various

diff --git a/pypy/module/_jitlog/interp_jitlog.py b/pypy/module/_jitlog/interp_jitlog.py
--- a/pypy/module/_jitlog/interp_jitlog.py
+++ b/pypy/module/_jitlog/interp_jitlog.py
@@ -12,7 +12,7 @@
 
 def JitlogError(space, e):
     w_JitlogError = space.fromcache(Cache).w_JitlogError
-    return OperationError(w_JitlogError, space.wrap(e.msg))
+    return OperationError(w_JitlogError, space.newtext(e.msg))
 
 @unwrap_spec(fileno=int)
 def enable(space, fileno):
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -47,7 +47,7 @@
 
 def VMProfError(space, e):
     w_VMProfError = space.fromcache(Cache).w_VMProfError
-    return OperationError(w_VMProfError, space.wrap(e.msg))
+    return OperationError(w_VMProfError, space.newtext(e.msg))
 
 
 @unwrap_spec(fileno=int, period=float)
@@ -60,9 +60,9 @@
     Must be smaller than 1.0
     """
     w_modules = space.sys.get('modules')
-    #if space.contains_w(w_modules, space.wrap('_continuation')):
-    #    space.warn(space.wrap("Using _continuation/greenlet/stacklet together "
-    #                          "with vmprof will crash"),
+    #if space.contains_w(w_modules, space.newtext('_continuation')):
+    #    space.warn(space.newtext("Using _continuation/greenlet/stacklet together "
+    #                             "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
         rvmprof.enable(fileno, period)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -53,15 +53,14 @@
         except KeyError:
             raise OperationError(space.w_KeyError, space.newtext(name))
         assert isinstance(w_zipimporter, W_ZipImporter)
-        w = space.wrap
         w_d = space.newdict()
         for key, info in w_zipimporter.zip_file.NameToInfo.iteritems():
             if ZIPSEP != os.path.sep:
                 key = key.replace(ZIPSEP, os.path.sep)
             space.setitem(w_d, space.newtext(key), space.newtuple([
-                w(info.filename), w(info.compress_type), w(info.compress_size),
-                w(info.file_size), w(info.file_offset), w(info.dostime),
-                w(info.dosdate), w(info.CRC)]))
+                space.newtext(info.filename), space.newint(info.compress_type), space.newint(info.compress_size),
+                space.newint(info.file_size), space.newint(info.file_offset), space.newint(info.dostime),
+                space.newint(info.dosdate), space.newint(info.CRC)]))
         return w_d
 
     def keys(self, space):
@@ -74,7 +73,6 @@
         return space.newlist(values_w)
 
     def items(self, space):
-        w = space.wrap
         items_w = [space.newtuple([space.newtext(key), self.getitem(space, key)])
                    for key in self.cache.keys()]
         return space.newlist(items_w)
@@ -145,7 +143,6 @@
             return fname
 
     def import_py_file(self, space, modname, filename, buf, pkgpath):
-        w = space.wrap
         w_mod = Module(space, space.newtext(modname))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
         space.setattr(w_mod, space.newtext('__loader__'), self)

From pypy.commits at gmail.com  Fri Nov 11 11:44:25 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:25 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: __spacebind__ -> spacebind
 (there's no need to make it look like a special method)
Message-ID: <5825f569.12111c0a.f0f31.224e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88319:a7bb81dcc53e
Date: 2016-11-10 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/a7bb81dcc53e/

Log:	__spacebind__ -> spacebind (there's no need to make it look like a
	special method)

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -50,7 +50,7 @@
     def __init__(self, fields):
         self.fields = fields
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         return space.newtuple([space.wrap(field) for field in self.fields])
 
 
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -437,7 +437,7 @@
     def __init__(self, fields):
         self.fields = fields
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         return space.newtuple([space.newtext(field) for field in self.fields])
 
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -313,7 +313,10 @@
         raise oefmt(space.w_TypeError,
                     "ord() expected string of length 1, but %T found", self)
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
+        """ Return a version of the object bound to a specific object space
+        instance. This is used for objects (like e.g. TypeDefs) that are
+        constructed before there is an object space instance. """
         return self
 
     def unwrap(self, space):
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -986,7 +986,7 @@
 
     # lazy binding to space
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         # we first make a real Function object out of it
         # and the result is a wrapped version of this Function.
         return self.get_function(space)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -417,7 +417,7 @@
     def __init__(self, function):
         self.function = function
 
-    def __spacebind__(self, space):
+    def spacebind(self, space):
         return self.function(space)
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -88,4 +88,4 @@
         raise oefmt(space.w_TypeError,
                     "don't know the size pointed to by '%s'", ctype.name)
     ptr = w_cdata.unsafe_escaping_ptr()    # w_cdata kept alive by MiniBuffer()
-    return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata))
+    return MiniBuffer(LLBuffer(ptr, size), w_cdata)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -297,7 +297,7 @@
         #
         # returns a single-argument function
         space = self.space
-        w_ffi = space.wrap(self)
+        w_ffi = self
         w_decorator = call_python.get_generic_decorator(space)
         return space.appexec([w_decorator, w_ffi, w_name, w_error, w_onerror],
         """(decorator, ffi, name, error, onerror):
@@ -405,7 +405,7 @@
                 result += ')'
             result += w_ctype.name[w_ctype.name_position:]
         # Python 3: bytes -> unicode string
-        return self.space.wrap(result)
+        return self.space.newtext(result)
 
 
     @unwrap_spec(code=int)
@@ -518,7 +518,7 @@
             _, offset = w_ctype.direct_typeoffsetof(w_field_or_array, False)
         else:
             offset = self._more_offsetof(w_ctype, w_field_or_array, args_w)
-        return self.space.wrap(offset)
+        return self.space.newint(offset)
 
 
     @unwrap_spec(w_cdata=W_CData, maxlen=int)
@@ -574,7 +574,7 @@
             if size < 0:
                 raise oefmt(self.w_FFIError,
                             "don't know the size of ctype '%s'", w_ctype.name)
-        return self.space.wrap(size)
+        return self.space.newint(size)
 
 
     def descr_typeof(self, w_arg):
@@ -642,7 +642,7 @@
         lst1_w = []
         for i in range(rffi.getintfield(ctx, 'c_num_typenames')):
             s = rffi.charp2str(ctx.c_typenames[i].c_name)
-            lst1_w.append(space.wrap(s))
+            lst1_w.append(space.newtext(s))
 
         lst2_w = []
         lst3_w = []
@@ -655,7 +655,7 @@
                 lst_w = lst3_w
             else:
                 lst_w = lst2_w
-            lst_w.append(space.wrap(s))
+            lst_w.append(space.newtext(s))
 
         return space.newtuple([space.newlist(lst1_w),
                                space.newlist(lst2_w),
@@ -734,7 +734,7 @@
     return r
 
 def W_FFIObject___new__(space, w_subtype, __args__):
-    return space.wrap(make_plain_ffi_object(space, w_subtype))
+    return make_plain_ffi_object(space, w_subtype)
 
 def make_CData(space):
     return space.gettypefor(W_CData)
@@ -744,7 +744,7 @@
 
 def make_NULL(space):
     ctvoidp = newtype._new_voidp_type(space)
-    w_NULL = ctvoidp.cast(space.wrap(0))
+    w_NULL = ctvoidp.cast(space.newint(0))
     return w_NULL
 
 def make_error(space):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1132,7 +1132,7 @@
         self.w_sslerror = space.new_exception_class(
             "_ssl.SSLError", w_socketerror)
         space.setattr(self.w_sslerror, space.newtext('__str__'),
-                      interp2app(SSLError_descr_str).__spacebind__(space))
+                      interp2app(SSLError_descr_str).spacebind(space))
         self.w_sslzeroreturnerror = space.new_exception_class(
             "_ssl.SSLZeroReturnError", self.w_sslerror)
         self.w_sslwantreaderror = space.new_exception_class(
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -154,7 +154,7 @@
         if isinstance(x, float):
             return W_FloatObject(x)
         if isinstance(x, W_Root):
-            w_result = x.__spacebind__(self)
+            w_result = x.spacebind(self)
             #print 'wrapping', x, '->', w_result
             return w_result
         if isinstance(x, base_int):

From pypy.commits at gmail.com  Fri Nov 11 11:44:27 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:27 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: reinstate accidentally killed
 test bits (there's still a bug somewhere in _cffi_backend)
Message-ID: <5825f56b.cf3fc20a.1e77d.e7a7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88320:1f45decc80d1
Date: 2016-11-10 18:55 +0100
http://bitbucket.org/pypy/pypy/changeset/1f45decc80d1/

Log:	reinstate accidentally killed test bits (there's still a bug
	somewhere in _cffi_backend)

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1203,6 +1203,20 @@
     orig_getline = linecache.getline
     try:
         linecache.getline = lambda *args: 'LINE'    # hack: speed up PyPy tests
+        sys.stderr = cStringIO.StringIO()
+        assert f(100) == 300
+        assert sys.stderr.getvalue() == ''
+        assert f(10000) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Traceback (most recent call last):
+  File "$", line $, in Zcb1
+    $
+  File "$", line $, in check_value
+    $
+ValueError: 42
+""")
+        sys.stderr = cStringIO.StringIO()
         bigvalue = 20000
         assert f(bigvalue) == -42
         assert matches(sys.stderr.getvalue(), """\
@@ -1210,6 +1224,59 @@
 Trying to convert the result back to C:
 OverflowError: integer 60000 does not fit 'short'
 """)
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        assert len(seen) == 0
+        assert ff(bigvalue) == -42
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = 81
+        assert ff(bigvalue) == 81
+        oops_result = None
+        assert sys.stderr.getvalue() == ""
+        assert len(seen) == 1
+        exc, val, tb = seen[0]
+        assert exc is OverflowError
+        assert str(val) == "integer 60000 does not fit 'short'"
+        #
+        sys.stderr = cStringIO.StringIO()
+        bigvalue = 20000
+        del seen[:]
+        oops_result = "xy"     # not None and not an int!
+        assert ff(bigvalue) == -42
+        oops_result = None
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+TypeError: $integer$
+""")
+        #
+        sys.stderr = cStringIO.StringIO()
+        seen = "not a list"    # this makes the oops() function crash
+        assert ff(bigvalue) == -42
+        assert matches(sys.stderr.getvalue(), """\
+From cffi callback <function$Zcb1 at 0x$>:
+Trying to convert the result back to C:
+OverflowError: integer 60000 does not fit 'short'
+
+During the call to 'onerror', another exception occurred:
+
+Traceback (most recent call last):
+  File "$", line $, in oops
+    $
+AttributeError: 'str' object has no attribute 'append'
+""")
     finally:
         sys.stderr = orig_stderr
         linecache.getline = orig_getline

From pypy.commits at gmail.com  Fri Nov 11 11:44:31 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:31 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix problem with tracebacks
 (space.wrap(w_obj-or-None) replaces the None with a
Message-ID: <5825f56f.ca57c20a.44793.e3e3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88322:ba3fe6222638
Date: 2016-11-11 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/ba3fe6222638/

Log:	fix problem with tracebacks (space.wrap(w_obj-or-None) replaces the
	None with a w_None, which I had missed in this case)

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -241,7 +241,7 @@
             if with_traceback:
                 w_t = self.w_type
                 w_v = self.get_w_value(space)
-                w_tb = self.get_traceback()
+                w_tb = self.get_w_traceback(space)
                 space.appexec([space.newtext(where),
                                space.newtext(objrepr),
                                space.newtext(extra_line),
@@ -286,6 +286,13 @@
             tb.frame.mark_as_escaped()
         return tb
 
+    def get_w_traceback(self, space):
+        """Return a traceback or w_None. """
+        tb = self.get_traceback()
+        if tb is None:
+            return space.w_None
+        return tb
+
     def set_traceback(self, traceback):
         """Set the current traceback."""
         self._application_traceback = traceback
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -323,7 +323,7 @@
             if operr is not None:
                 w_value = operr.get_w_value(space)
                 w_arg = space.newtuple([operr.w_type, w_value,
-                                        operr.get_traceback()])
+                                        operr.get_w_traceback(space)])
 
             d = frame.getorcreatedebug()
             if d.w_locals is not None:
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -118,7 +118,7 @@
         operationerr.normalize_exception(space)
         w_type = operationerr.w_type
         w_value = operationerr.get_w_value(space)
-        w_traceback = space.wrap(operationerr.get_traceback())
+        w_traceback = operationerr.get_w_traceback(space)
 
         # for debugging convenience we also insert the exception into
         # the interpreter-level sys.last_xxx
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -443,7 +443,7 @@
             w_tb = space.w_None
         else:
             w_exc_value = self.last_exception.get_w_value(space)
-            w_tb = w(self.last_exception.get_traceback())
+            w_tb = self.last_exception.get_w_traceback(space)
 
         d = self.getorcreatedebug()
         tup_state = [
@@ -858,7 +858,7 @@
             while f is not None and f.last_exception is None:
                 f = f.f_backref()
             if f is not None:
-                return f.last_exception.get_traceback()
+                return f.last_exception.get_w_traceback(space)
         return space.w_None
 
     def fget_f_restricted(self, space):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1148,7 +1148,7 @@
             # app-level exception
             operr = w_unroller.operr
             self.last_exception = operr
-            w_traceback = operr.get_traceback()
+            w_traceback = operr.get_w_traceback(self.space)
             w_suppress = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 operr.w_type,
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -73,7 +73,7 @@
     frame hidden from applevel.
     """
     operr = space.getexecutioncontext().sys_exc_info(for_hidden=True)
-    return space.w_None if operr is None else operr.get_traceback()
+    return space.w_None if operr is None else operr.get_w_traceback(space)
 
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -183,7 +183,7 @@
                 e.normalize_exception(space)
                 w_t = e.w_type
                 w_v = e.get_w_value(space)
-                w_tb = space.wrap(e.get_traceback())
+                w_tb = e.get_w_traceback(space)
                 w_res = space.call_function(self.w_onerror, w_t, w_v, w_tb)
                 if not space.is_none(w_res):
                     self.convert_result(ll_res, w_res)
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -60,7 +60,7 @@
                         break
             unwrap_value(space, write_ptr, ptr, 0, letter, w_res)
     except OperationError as e:
-        tbprint(space, e.get_traceback(),
+        tbprint(space, e.get_w_traceback(space),
                 space.newtext(e.errorstr(space)))
         # force the result to be zero
         if callback_ptr.result is not None:
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -58,7 +58,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, operror.get_traceback())
+        ptraceback[0] = make_ref(space, operror.get_w_traceback(space))
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
@@ -286,7 +286,7 @@
 
     w_type = operror.w_type
     w_value = operror.get_w_value(space)
-    w_tb = operror.get_traceback()
+    w_tb = operror.get_w_traceback(space)
 
     if rffi.cast(lltype.Signed, set_sys_last_vars):
         space.sys.setdictvalue(space, "last_type", w_type)
@@ -373,7 +373,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, operror.get_traceback())
+        ptraceback[0] = make_ref(space, operror.get_w_traceback(space))
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -161,7 +161,7 @@
             if operror is None:
                 return space.w_None
             else:
-                return operror.get_traceback()
+                return operror.get_w_traceback(space)
         return None
 
     def get_w_default_encoder(self):
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -98,7 +98,7 @@
         return space.newtuple([space.w_None, space.w_None, space.w_None])
     else:
         return space.newtuple([operror.w_type, operror.get_w_value(space),
-                               operror.get_traceback()])
+                               operror.get_w_traceback(space)])
 
 def exc_info_without_tb(space, frame):
     operror = frame.last_exception

From pypy.commits at gmail.com  Fri Nov 11 11:44:32 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:32 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: a few more in _cffi_backend
Message-ID: <5825f570.c5311c0a.74061.1f08@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88323:97de008631bc
Date: 2016-11-11 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/97de008631bc/

Log:	a few more in _cffi_backend

diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py
--- a/pypy/module/_cffi_backend/call_python.py
+++ b/pypy/module/_cffi_backend/call_python.py
@@ -130,4 +130,4 @@
 
 @specialize.memo()
 def get_generic_decorator(space):
-    return space.wrap(interp2app(externpy_deco)) # init time
+    return interp2app(externpy_deco).spacebind(space)
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -97,7 +97,7 @@
 
     def _fget(self, attrchar):
         if attrchar == 'i':     # item
-            return self.space.wrap(self.ctitem)
+            return self.ctitem
         if attrchar == 'l':     # length
             if self.length >= 0:
                 return self.space.newint(self.length)
diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py
--- a/pypy/module/_cffi_backend/ctypeenum.py
+++ b/pypy/module/_cffi_backend/ctypeenum.py
@@ -23,15 +23,15 @@
             space = self.space
             w_dct = space.newdict()
             for enumvalue, enumerator in self.enumvalues2erators.iteritems():
-                space.setitem(w_dct, space.wrap(enumvalue),
-                                     space.wrap(enumerator))
+                space.setitem(w_dct, space.newint(enumvalue),
+                                     space.newtext(enumerator))
             return w_dct
         if attrchar == 'R':     # relements
             space = self.space
             w_dct = space.newdict()
             for enumerator, enumvalue in self.enumerators2values.iteritems():
-                space.setitem(w_dct, space.wrap(enumerator),
-                                     space.wrap(enumvalue))
+                space.setitem(w_dct, space.newtext(enumerator),
+                                     space.newint(enumvalue))
             return w_dct
         return self._super._fget(self, attrchar)
 
@@ -51,7 +51,7 @@
             s = self.enumvalues2erators[value]
         except KeyError:
             s = str(value)
-        return self.space.wrap(s)
+        return self.space.newtext(s)
 
 
 class W_CTypeEnumSigned(_Mixin_Enum, W_CTypePrimitiveSigned):
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -117,12 +117,11 @@
 
     def _fget(self, attrchar):
         if attrchar == 'a':    # args
-            return self.space.newtuple([self.space.wrap(a)
-                                        for a in self.fargs])
+            return self.space.newtuple([a for a in self.fargs])
         if attrchar == 'r':    # result
             return self.ctitem
         if attrchar == 'E':    # ellipsis
-            return self.space.wrap(self.ellipsis)
+            return self.space.newbool(self.ellipsis)
         if attrchar == 'A':    # abi
             return self.space.newint(self.abi)
         return W_CTypePtrBase._fget(self, attrchar)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -262,7 +262,7 @@
         #
         w_ctype = self.ffi_type(w_arg, ACCEPT_ALL)
         align = w_ctype.alignof()
-        return self.space.wrap(align)
+        return self.space.newint(align)
 
 
     @unwrap_spec(w_cdata=W_CData, size=int)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -56,7 +56,7 @@
 @unwrap_spec(w_ctype=ctypeobj.W_CType, following=int)
 def typeoffsetof(space, w_ctype, w_field_or_index, following=0):
     ctype, offset = w_ctype.direct_typeoffsetof(w_field_or_index, following)
-    return space.newtuple([space.wrap(ctype), space.newint(offset)])
+    return space.newtuple([ctype, space.newint(offset)])
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType, w_cdata=cdataobj.W_CData, offset=int)
 def rawaddressof(space, w_ctype, w_cdata, offset):
@@ -99,7 +99,7 @@
             break
         key = rffi.charp2str(p)
         value = rffi.charp2str(rffi.ptradd(p, len(key) + 1))
-        space.setitem_str(w_dict, key, space.wrap(value))
+        space.setitem_str(w_dict, key, space.newtext(value))
         index += 1
 
 # ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -243,7 +243,7 @@
                 if (op == cffi_opcode.OP_GLOBAL_VAR or
                     op == cffi_opcode.OP_GLOBAL_VAR_F):
                     continue
-            names_w.append(space.wrap(rffi.charp2str(g[i].c_name)))
+            names_w.append(space.newtext(rffi.charp2str(g[i].c_name)))
         return space.newlist(names_w)
 
     def full_dict_copy(self):
@@ -252,7 +252,7 @@
         g = self.ctx.c_globals
         w_result = space.newdict()
         for i in range(total):
-            w_attr = space.wrap(rffi.charp2str(g[i].c_name))
+            w_attr = space.newtext(rffi.charp2str(g[i].c_name))
             w_value = self._get_attr(w_attr)
             space.setitem(w_result, w_attr, w_value)
         return w_result
@@ -261,7 +261,7 @@
         # rebuild a string object from 'varname', to do typechecks and
         # to force a unicode back to a plain string
         space = self.space
-        w_value = self._get_attr(space.wrap(varname))
+        w_value = self._get_attr(space.newtext(varname))
         if isinstance(w_value, cglob.W_GlobSupport):
             # regular case: a global variable
             return w_value.address()

From pypy.commits at gmail.com  Fri Nov 11 11:44:29 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 Nov 2016 08:44:29 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: call spacebind explicitly here
Message-ID: <5825f56d.2636c20a.6c87.df05@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88321:788a8ad341f2
Date: 2016-11-11 12:56 +0100
http://bitbucket.org/pypy/pypy/changeset/788a8ad341f2/

Log:	call spacebind explicitly here

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -274,7 +274,8 @@
     for error in ("strict", "ignore", "replace", "xmlcharrefreplace",
                   "backslashreplace"):
         name = error + "_errors"
-        state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
+        state.codec_error_registry[error] = interp2app(
+                globals()[name]).spacebind(space)
 
 
 @unwrap_spec(errors=str)

From pypy.commits at gmail.com  Fri Nov 11 13:27:32 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 11 Nov 2016 10:27:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Copy attributes when calling
 memoryview(<memoryview>)
Message-ID: <58260d94.94841c0a.ed79.856f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88324:3ccfd8d8c12b
Date: 2016-11-11 18:26 +0000
http://bitbucket.org/pypy/pypy/changeset/3ccfd8d8c12b/

Log:	Copy attributes when calling memoryview(<memoryview>)

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -80,6 +80,9 @@
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
+        if isinstance(w_object, W_MemoryView):
+            w_object._check_released(space)
+            return W_MemoryView.copy(w_object)
         return W_MemoryView(*space.buffer_w_ex(w_object, space.BUF_FULL_RO))
 
     def _make_descr__cmp(name):

From pypy.commits at gmail.com  Fri Nov 11 14:13:04 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 11 Nov 2016 11:13:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Give correct error message for unicode
 array initializers and remove confusing special case
Message-ID: <58261840.8f95c20a.34d91.2460@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88325:8dd2ae08247a
Date: 2016-11-11 19:12 +0000
http://bitbucket.org/pypy/pypy/changeset/8dd2ae08247a/

Log:	Give correct error message for unicode array initializers and remove
	confusing special case

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -41,6 +41,18 @@
 
     if len(__args__.arguments_w) > 0:
         w_initializer = __args__.arguments_w[0]
+        if tc != 'u':
+            if space.isinstance_w(w_initializer, space.w_unicode):
+                raise oefmt(
+                    space.w_TypeError,
+                    "cannot use a str to initialize an array with "
+                    "typecode '%s'", tc)
+            elif (isinstance(w_initializer, W_ArrayBase)
+                    and w_initializer.typecode == 'u'):
+                raise oefmt(
+                    space.w_TypeError,
+                    "cannot use a unicode array to initialize an array with "
+                    "typecode '%s'", tc)
         if isinstance(w_initializer, W_ArrayBase):
             a.extend(w_initializer, True)
         elif space.type(w_initializer) is space.w_list:
@@ -719,11 +731,7 @@
                     except OperationError as e:
                         if e.async(space):
                             raise
-                        if space.isinstance_w(w_item, space.w_unicode):
-                            msg = ("cannot use a str to initialize an array"
-                                   " with typecode '" + mytype.typecode + "'")
-                        else:
-                            msg = "array item must be " + mytype.unwrap[:-2]
+                        msg = "array item must be " + mytype.unwrap[:-2]
                         raise OperationError(space.w_TypeError, space.wrap(msg))
                 else:
                     raise

From pypy.commits at gmail.com  Fri Nov 11 18:14:07 2016
From: pypy.commits at gmail.com (stefanor)
Date: Fri, 11 Nov 2016 15:14:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Respect --no-profopt,
 so you can do -O3 --shared
Message-ID: <582650bf.838e1c0a.31030.ad91@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r88326:c47d09153826
Date: 2016-11-11 23:13 +0000
http://bitbucket.org/pypy/pypy/changeset/c47d09153826/

Log:	Respect --no-profopt, so you can do -O3 --shared

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -253,7 +253,8 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if config.translation.profopt is not None:
+            if (config.translation.profopt is not None
+                    and not config.translation.noprofopt):
                 raise Exception("Cannot use the --profopt option "
                                 "when --shared is on (it is by default). "
                                 "See issue #2398.")

From pypy.commits at gmail.com  Sat Nov 12 03:16:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 00:16:50 -0800 (PST)
Subject: [pypy-commit] pypy default: dirfd might be a macro on OS/X
Message-ID: <5826cff2.61adc20a.cc8be.e32c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88327:159f3934b735
Date: 2016-11-12 09:16 +0100
http://bitbucket.org/pypy/pypy/changeset/159f3934b735/

Log:	dirfd might be a macro on OS/X

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -655,7 +655,8 @@
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
-    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False,
+                       macro=True)
     c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
                          save_err=rffi.RFFI_SAVE_ERRNO)
 else:

From pypy.commits at gmail.com  Sat Nov 12 03:20:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 00:20:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5826d0b3.88711c0a.1ea76.d59b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88328:8cb801603f8a
Date: 2016-11-12 09:19 +0100
http://bitbucket.org/pypy/pypy/changeset/8cb801603f8a/

Log:	hg merge default

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -358,7 +358,7 @@
 
 /**********  PyPy-specific section  **********/
 
-void _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
+PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]);   /* forward */
 
 static struct _cffi_pypy_init_s {
     const char *name;
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -392,7 +392,7 @@
         #
         # the init function
         prnt('#ifdef PYPY_VERSION')
-        prnt('void')
+        prnt('PyMODINIT_FUNC')
         prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
         prnt('{')
         if self._num_externpy:
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -407,3 +407,18 @@
 long and gnarly. On the other hand, all our infrastructure (buildbots,
 benchmarking, etc) would have to be adapted. So unless somebody steps up and
 volunteers to do all that work, it will likely not happen.
+
+
+What is needed for Windows 64 support of PyPy?
+-----------------------------------------------
+
+First, please note that the Windows 32 PyPy binary works just fine on Windows
+64. The only problem is that it only supports up to 4GB of heap per process.
+
+As to real Windows 64 support: Currently we don't have an active PyPy developer
+whose main development platform is Windows. So if you are interested in getting
+Windows 64 support, we encourage you to volunteer `to make it happen`_! Another
+option would be to pay some PyPy developers to implement Windows 64 support,
+but so far there doesn't seem to be an overwhelming commercial interest in it.
+
+.. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -11,7 +11,10 @@
 
 Work proceeds at a good pace on the PyPy3.5
 version due to a grant_ from the Mozilla Foundation, and some of those
-changes have been backported to PyPy2.7 where relevant
+changes have been backported to PyPy2.7 where relevant.
+
+The PowerPC and s390x backend have been enhanced_ with the capability use SIMD instructions
+ for micronumpy loops.
 
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
@@ -45,6 +48,7 @@
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+.. _`enhanced`: https://morepypy.blogspot.co.at/2016/11/vectorization-extended-powerpc-and-s390x.html
 
 What is PyPy?
 =============
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -260,7 +260,8 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if config.translation.profopt is not None:
+            if (config.translation.profopt is not None
+                    and not config.translation.noprofopt):
                 raise Exception("Cannot use the --profopt option "
                                 "when --shared is on (it is by default). "
                                 "See issue #2398.")
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -638,20 +638,6 @@
     w_exc = new_import_error(space, w_msg, w_name, w_path)
     raise OperationError(space.w_ImportError, w_exc)
 
-def _convert_unexpected_exception_extra(space, e):
-    "NOT_RPYTHON"
-    if e.__class__.__name__ in (
-        'Skipped',     # list of exception class names that are ok
-        ):             # to get during ==untranslated tests== only
-        raise
-    # include the RPython-level traceback
-    exc = sys.exc_info()
-    import traceback, cStringIO
-    f = cStringIO.StringIO()
-    print >> f, "\nTraceback (interpreter-level):"
-    traceback.print_tb(exc[2], file=f)
-    return f.getvalue()
-
 @jit.dont_look_inside
 def get_converted_unexpected_exception(space, e):
     """This is used in two places when we get an non-OperationError
@@ -689,7 +675,9 @@
             debug_print_traceback()
             extra = '; internal traceback was dumped to stderr'
         else:
-            extra = _convert_unexpected_exception_extra(space, e)
+            # when untranslated, we don't wrap into an app-level
+            # SystemError (this makes debugging tests harder)
+            raise
         return OperationError(space.w_SystemError, space.wrap(
             "unexpected internal exception (please report a bug): %r%s" %
             (e, extra)))
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -805,6 +805,8 @@
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
     def test_system_error(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
         class UnexpectedException(Exception):
             pass
         space = self.space
@@ -820,6 +822,8 @@
                 'report a bug): UnexpectedException') in err
 
     def test_system_error_2(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
         class UnexpectedException(Exception):
             pass
         space = self.space
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -73,7 +73,6 @@
         from __pypy__ import do_what_I_mean
         x = do_what_I_mean()
         assert x == 42
-        raises(SystemError, do_what_I_mean, 1)
 
     def test_list_strategy(self):
         from __pypy__ import strategy
@@ -132,9 +131,19 @@
 class AppTestJitFeatures(object):
     spaceconfig = {"translation.jit": True}
 
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
     def test_jit_backend_features(self):
         from __pypy__ import jit_backend_features
         supported_types = jit_backend_features
         assert isinstance(supported_types, list)
         for x in supported_types:
             assert x in ['floats', 'singlefloats', 'longlong']
+
+    def test_do_what_I_mean_error(self):
+        if not self.runappdirect:
+            skip("we don't wrap a random exception inside SystemError "
+                 "when untranslated, because it makes testing harder")
+        from __pypy__ import do_what_I_mean
+        raises(SystemError, do_what_I_mean, 1)
diff --git a/pypy/module/pypyjit/test/conftest.py b/pypy/module/pypyjit/test/conftest.py
--- a/pypy/module/pypyjit/test/conftest.py
+++ b/pypy/module/pypyjit/test/conftest.py
@@ -1,5 +1,7 @@
 def pytest_addoption(parser):
     group = parser.getgroup("pypyjit options")
     group.addoption("--pypy", action="store", default=None, dest="pypy_c",
-                    help="the location of the JIT enabled pypy-c")
-
+                    help="DEPRECATED: use this in test_pypy_c instead")
+# XXX kill the corresponding section in the buildbot run,
+# which (as far as I can tell) ignores that option entirely and does
+# the same as the regular py.test.
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,8 +1,8 @@
 # Edit these appropriately before running this script
 maj=5
-min=4
-rev=1
-branchname=release-$maj.x  # ==OR== release-$maj.$min.x
+min=6
+rev=0
+branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
 tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
@@ -39,7 +39,7 @@
 zip -rq $rel-$plat.zip $rel-$plat
 rm -rf $rel-$plat
 
-# Do this after creating a tag, note the untarred directory is pypy-pypy-<hash>
+# Requires a valid $tagname, note the untarred directory is pypy-pypy-<hash>
 # so make sure there is not another one
 wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
 tar -xf $tagname.tar.bz2
diff --git a/rpython/jit/backend/zarch/instructions.py b/rpython/jit/backend/zarch/instructions.py
--- a/rpython/jit/backend/zarch/instructions.py
+++ b/rpython/jit/backend/zarch/instructions.py
@@ -76,7 +76,6 @@
     'CLGRJ':   ('rie_b',  ['\xEC','\x65']),
     # compare and swap
     'CSG':     ('rsy_a', ['\xEB','\x30']),
-
 }
 
 logic_mnemonic_codes = {
@@ -340,7 +339,7 @@
     'VLVG':  ('vrs_b', ['\xE7','\x22']),
     'VLGV':  ('vrs_c', ['\xE7','\x21']),
 
-    # '': ('', ['','']),
+    'VSEG':  ('vrr_a', ['\xE7','\x5F'], 'v,v,m'),
 }
 
 all_mnemonic_codes.update(arith_mnemonic_codes)
diff --git a/rpython/jit/backend/zarch/vector_ext.py b/rpython/jit/backend/zarch/vector_ext.py
--- a/rpython/jit/backend/zarch/vector_ext.py
+++ b/rpython/jit/backend/zarch/vector_ext.py
@@ -139,6 +139,8 @@
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(0), l.itemsize_to_mask(nsize))
             self.mc.VLGV(r.SCRATCH, loc0, l.addr(1), l.itemsize_to_mask(osize))
             self.mc.VLVG(resloc, r.SCRATCH, l.addr(1), l.itemsize_to_mask(nsize))
+            if nsize == 8:
+                self.mc.VSEG(resloc, resloc, l.itemsize_to_mask(osize))
 
     def emit_vec_float_abs(self, op, arglocs, regalloc):
         resloc, argloc, sizeloc = arglocs
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -14,6 +14,7 @@
 from rpython.jit.metainterp.jitexc import NotAVectorizeableLoop, NotAProfitableLoop
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.debug import debug_print
 
 
 def forwarded_vecinfo(op):
@@ -315,11 +316,8 @@
 
 def failnbail_transformation(msg):
     msg = '%s\n' % msg
-    if we_are_translated():
-        llop.debug_print(lltype.Void, msg)
-    else:
-        import pdb; pdb.set_trace()
-    raise NotImplementedError(msg)
+    debug_print(msg)
+    raise NotAVectorizeableLoop
 
 def turn_into_vector(state, pack):
     """ Turn a pack into a vector instruction """
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -860,6 +860,37 @@
         free_raw_storage(vb)
         free_raw_storage(vc)
 
+    def test_float_int32_casts(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        def f(bytecount, va, vb, vc):
+            i = 0
+            j = 0
+            while j < bytecount:
+                myjitdriver.jit_merge_point()
+                a = raw_storage_getitem(rffi.DOUBLE,va,j)
+                b = raw_storage_getitem(rffi.INT,vb,i)
+                c = a+rffi.cast(rffi.DOUBLE,b)
+                raw_storage_setitem(vc, j, c)
+                i += 4
+                j += 8
+
+        count = 32
+        va = alloc_raw_storage(8*count, zero=True)
+        vb = alloc_raw_storage(4*count, zero=True)
+        for i,v in enumerate([1.0,2.0,3.0,4.0]*(count/4)):
+            raw_storage_setitem(va, i*8, rffi.cast(rffi.DOUBLE,v))
+        for i,v in enumerate([-1,-2,-3,-4]*(count/4)):
+            raw_storage_setitem(vb, i*4, rffi.cast(rffi.INT,v))
+        vc = alloc_raw_storage(8*count, zero=True)
+        self.meta_interp(f, [8*count, va, vb, vc], vec=True)
+
+        for i in range(count):
+            assert raw_storage_getitem(rffi.DOUBLE,vc,i*8) == 0.0
+
+        free_raw_storage(va)
+        free_raw_storage(vb)
+        free_raw_storage(vc)
+
 
 class TestLLtype(LLJitMixin, VectorizeTests):
     # skip some tests on this backend
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -323,93 +323,165 @@
 
     def __add__(self, other):
         x = long(self)
-        y = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return x + other
+        y = long(other)
         return self._widen(other, x + y)
-    __radd__ = __add__
+
+    def __radd__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other + x
+        y = long(other)
+        return self._widen(other, x + y)
 
     def __sub__(self, other):
         x = long(self)
-        y = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return x - other
+        y = long(other)
         return self._widen(other, x - y)
 
     def __rsub__(self, other):
         y = long(self)
-        x = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return other - y
+        x = long(other)
         return self._widen(other, x - y)
 
     def __mul__(self, other):
         x = long(self)
-        y = other      # may be a float
+        if not isinstance(other, (int, long)):
+            return x * other
+        y = long(other)
         return self._widen(other, x * y)
-    __rmul__ = __mul__
+
+    def __rmul__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other * x
+        y = long(other)
+        return self._widen(other, x * y)
 
     def __div__(self, other):
         x = long(self)
-        y = other      # may be a float
-        return self._widen(other, x / y)
-
-    __floordiv__ = __div__
+        if not isinstance(other, (int, long)):
+            return x / other
+        y = long(other)
+        return self._widen(other, x // y)
 
     def __rdiv__(self, other):
         y = long(self)
-        x = other      # may be a float
-        return self._widen(other, x / y)
+        if not isinstance(other, (int, long)):
+            return other / y
+        x = long(other)
+        return self._widen(other, x // y)
 
-    __rfloordiv__ = __rdiv__
+    def __floordiv__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return x // other
+        y = long(other)
+        return self._widen(other, x // y)
+
+    def __rfloordiv__(self, other):
+        y = long(self)
+        if not isinstance(other, (int, long)):
+            return other // y
+        x = long(other)
+        return self._widen(other, x // y)
 
     def __mod__(self, other):
         x = long(self)
-        y = other      # not rpython if it is a float
+        if not isinstance(other, (int, long)):
+            return x % other
+        y = long(other)
         return self._widen(other, x % y)
 
     def __rmod__(self, other):
         y = long(self)
-        x = other      # not rpython if it is a float
+        if not isinstance(other, (int, long)):
+            return other % y
+        x = long(other)
         return self._widen(other, x % y)
 
     def __divmod__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return divmod(x, other)
         y = long(other)
         res = divmod(x, y)
-        return (self.__class__(res[0]), self.__class__(res[1]))
+        return (self._widen(other, res[0]), self._widen(other, res[1]))
 
     def __lshift__(self, n):
         x = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         y = long(n)
         return self.__class__(x << y)
 
     def __rlshift__(self, n):
         y = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         x = long(n)
-        return self._widen(n, x << y)
+        return n.__class__(x << y)
 
     def __rshift__(self, n):
         x = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         y = long(n)
-        return self._widen(n, x >> y)
+        return self.__class__(x >> y)
 
     def __rrshift__(self, n):
         y = long(self)
+        if not isinstance(n, (int, long)):
+            raise TypeError
         x = long(n)
-        return self._widen(n, x >> y)
+        return n.__class__(x >> y)
 
     def __or__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x | other
         y = long(other)
         return self._widen(other, x | y)
-    __ror__ = __or__
+
+    def __ror__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other | x
+        y = long(other)
+        return self._widen(other, x | y)
 
     def __and__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x & other
         y = long(other)
         return self._widen(other, x & y)
-    __rand__ = __and__
+
+    def __rand__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other & x
+        y = long(other)
+        return self._widen(other, x & y)
 
     def __xor__(self, other):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return x ^ other
         y = long(other)
         return self._widen(other, x ^ y)
-    __rxor__ = __xor__
+
+    def __rxor__(self, other):
+        x = long(self)
+        if not isinstance(other, (int, long)):
+            return other ^ x
+        y = long(other)
+        return self._widen(other, x ^ y)
 
     def __neg__(self):
         x = long(self)
@@ -428,12 +500,16 @@
 
     def __pow__(self, other, m=None):
         x = long(self)
+        if not isinstance(other, (int, long)):
+            return pow(x, other, m)
         y = long(other)
         res = pow(x, y, m)
         return self._widen(other, res)
 
     def __rpow__(self, other, m=None):
         y = long(self)
+        if not isinstance(other, (int, long)):
+            return pow(other, y, m)
         x = long(other)
         res = pow(x, y, m)
         return self._widen(other, res)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -655,7 +655,8 @@
     c_readdir = external('readdir', [DIRP], DIRENTP,
                          macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
     c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False)
-    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False)
+    c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False,
+                       macro=True)
     c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT,
                          save_err=rffi.RFFI_SAVE_ERRNO)
 else:
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -603,11 +603,75 @@
         assert r_uint64(self._64_umax) + r_uint64(1) == r_uint64(0)
         assert r_uint64(0) - r_uint64(1) == r_uint64(self._64_umax)
 
-    def test_operation_with_float(self):
-        def f(x):
-            assert r_longlong(x) + 0.5 == 43.5
-            assert r_longlong(x) - 0.5 == 42.5
-            assert r_longlong(x) * 0.5 == 21.5
-            assert r_longlong(x) / 0.8 == 53.75
-        f(43)
-        interpret(f, [43])
+
+def test_operation_with_float():
+    def f(x):
+        assert r_longlong(x) + 0.5 == 43.5
+        assert r_longlong(x) - 0.5 == 42.5
+        assert r_longlong(x) * 0.5 == 21.5
+        assert r_longlong(x) / 0.8 == 53.75
+    f(43)
+    interpret(f, [43])
+
+def test_int64_plus_int32():
+    assert r_uint64(1234567891234) + r_uint32(1) == r_uint64(1234567891235)
+
+def test_fallback_paths():
+
+    def make(pattern):
+        def method(self, other, *extra):
+            if extra:
+                assert extra == (None,)   # for 'pow'
+            if type(other) is long:
+                return pattern % other
+            else:
+                return NotImplemented
+        return method
+
+    class A(object):
+        __add__  = make("a+%d")
+        __radd__ = make("%d+a")
+        __sub__  = make("a-%d")
+        __rsub__ = make("%d-a")
+        __mul__  = make("a*%d")
+        __rmul__ = make("%d*a")
+        __div__  = make("a/%d")
+        __rdiv__ = make("%d/a")
+        __floordiv__  = make("a//%d")
+        __rfloordiv__ = make("%d//a")
+        __mod__  = make("a%%%d")
+        __rmod__ = make("%d%%a")
+        __and__  = make("a&%d")
+        __rand__ = make("%d&a")
+        __or__   = make("a|%d")
+        __ror__  = make("%d|a")
+        __xor__  = make("a^%d")
+        __rxor__ = make("%d^a")
+        __pow__  = make("a**%d")
+        __rpow__ = make("%d**a")
+
+    a = A()
+    assert r_uint32(42) + a == "42+a"
+    assert a + r_uint32(42) == "a+42"
+    assert r_uint32(42) - a == "42-a"
+    assert a - r_uint32(42) == "a-42"
+    assert r_uint32(42) * a == "42*a"
+    assert a * r_uint32(42) == "a*42"
+    assert r_uint32(42) / a == "42/a"
+    assert a / r_uint32(42) == "a/42"
+    assert r_uint32(42) // a == "42//a"
+    assert a // r_uint32(42) == "a//42"
+    assert r_uint32(42) % a == "42%a"
+    assert a % r_uint32(42) == "a%42"
+    py.test.raises(TypeError, "a << r_uint32(42)")
+    py.test.raises(TypeError, "r_uint32(42) << a")
+    py.test.raises(TypeError, "a >> r_uint32(42)")
+    py.test.raises(TypeError, "r_uint32(42) >> a")
+    assert r_uint32(42) & a == "42&a"
+    assert a & r_uint32(42) == "a&42"
+    assert r_uint32(42) | a == "42|a"
+    assert a | r_uint32(42) == "a|42"
+    assert r_uint32(42) ^ a == "42^a"
+    assert a ^ r_uint32(42) == "a^42"
+    assert r_uint32(42) ** a == "42**a"
+    assert a ** r_uint32(42) == "a**42"

From pypy.commits at gmail.com  Sat Nov 12 04:09:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 01:09:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix for calls to '__x86.get_pc_thunk'
Message-ID: <5826dc2d.876ec20a.9fe9f.ebb3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88329:71779cad4a61
Date: 2016-11-12 10:08 +0100
http://bitbucket.org/pypy/pypy/changeset/71779cad4a61/

Log:	Fix for calls to '__x86.get_pc_thunk'

diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -926,6 +926,13 @@
                 assert  lineoffset in (1,2)
                 return [InsnStackAdjust(-4)]
 
+        if target.startswith('__x86.get_pc_thunk.'):
+            # special case, found on x86-32: these static functions
+            # contain only a simple load of some non-GC pointer to
+            # a specific register (not necessarily EAX)
+            reg = '%e' + target.split('.')[-1]
+            return [InsnSetLocal(reg)]
+
         insns = [InsnCall(target, self.currentlineno),
                  InsnSetLocal(self.EAX)]      # the result is there
         if self.format in ('mingw32', 'msvc'):

From pypy.commits at gmail.com  Sat Nov 12 04:10:55 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 01:10:55 -0800 (PST)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Fix for calls to
 '__x86.get_pc_thunk'
Message-ID: <5826dc9f.44941c0a.9f413.25be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r88330:cf97314aa80f
Date: 2016-11-12 10:08 +0100
http://bitbucket.org/pypy/pypy/changeset/cf97314aa80f/

Log:	Fix for calls to '__x86.get_pc_thunk'

diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -926,6 +926,13 @@
                 assert  lineoffset in (1,2)
                 return [InsnStackAdjust(-4)]
 
+        if target.startswith('__x86.get_pc_thunk.'):
+            # special case, found on x86-32: these static functions
+            # contain only a simple load of some non-GC pointer to
+            # a specific register (not necessarily EAX)
+            reg = '%e' + target.split('.')[-1]
+            return [InsnSetLocal(reg)]
+
         insns = [InsnCall(target, self.currentlineno),
                  InsnSetLocal(self.EAX)]      # the result is there
         if self.format in ('mingw32', 'msvc'):

From pypy.commits at gmail.com  Sat Nov 12 05:00:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 02:00:08 -0800 (PST)
Subject: [pypy-commit] cffi default: Windows compilation fix
Message-ID: <5826e828.973f1c0a.c4df5.efcd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2812:b40175801569
Date: 2016-11-12 10:59 +0100
http://bitbucket.org/cffi/cffi/changeset/b40175801569/

Log:	Windows compilation fix

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2153,6 +2153,7 @@
 static PyObject *
 cdata_slice(CDataObject *cd, PySliceObject *slice)
 {
+    char *cdata;
     Py_ssize_t bounds[2];
     CTypeDescrObject *ct = _cdata_getslicearg(cd, slice, bounds);
     if (ct == NULL)
@@ -2165,7 +2166,7 @@
     }
     ct = (CTypeDescrObject *)ct->ct_stuff;
 
-    char *cdata = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0];
+    cdata = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0];
     return new_sized_cdata(cdata, ct, bounds[1]);
 }
 

From pypy.commits at gmail.com  Sat Nov 12 07:34:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 04:34:31 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update 2.7.10 to 2.7.12 (thanks
 Tiberium). Don't regenerate the html now
Message-ID: <58270c57.4948c20a.1b4e1.56e9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r816:07136ac2d239
Date: 2016-11-12 13:31 +0100
http://bitbucket.org/pypy/pypy.org/changeset/07136ac2d239/

Log:	update 2.7.10 to 2.7.12 (thanks Tiberium). Don't regenerate the html
	now

diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -3,7 +3,7 @@
 title: Python compatibility
 ---
 
-PyPy implements the Python language version 2.7.10. It supports all of the core
+PyPy implements the Python language version 2.7.12. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python `standard library modules`_; details below.
diff --git a/source/features.txt b/source/features.txt
--- a/source/features.txt
+++ b/source/features.txt
@@ -10,7 +10,7 @@
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
 
-**PyPy** implements **Python 2.7.10**.
+**PyPy** implements **Python 2.7.12**.
 It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
 in newer versions). It supports most of the commonly used Python
diff --git a/source/index.txt b/source/index.txt
--- a/source/index.txt
+++ b/source/index.txt
@@ -4,7 +4,7 @@
 ---
 
 PyPy is a `fast`_, `compliant`_ alternative implementation of the `Python`_
-language (2.7.10 and 3.3.5). It has several advantages and distinct features:
+language (2.7.12 and 3.3.5). It has several advantages and distinct features:
 
   * **Speed:** thanks to its Just-in-Time compiler, Python programs
     often run `faster`_ on PyPy.  `(What is a JIT compiler?)`_

From pypy.commits at gmail.com  Sat Nov 12 08:35:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:35:47 -0800 (PST)
Subject: [pypy-commit] cffi release-1.9: Release branch
Message-ID: <58271ab3.8f95c20a.34d91.4635@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.9
Changeset: r2813:ecfa510f0e93
Date: 2016-11-12 14:31 +0100
http://bitbucket.org/cffi/cffi/changeset/ecfa510f0e93/

Log:	Release branch


From pypy.commits at gmail.com  Sat Nov 12 08:35:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:35:49 -0800 (PST)
Subject: [pypy-commit] cffi release-1.9: md5/sha
Message-ID: <58271ab5.ca06c20a.15e6a.5d1e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.9
Changeset: r2814:9985415df9c9
Date: 2016-11-12 14:35 +0100
http://bitbucket.org/cffi/cffi/changeset/9985415df9c9/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.9.0.tar.gz
 
-   - MD5: ...
+   - MD5: 3806f3b4e093fa84fda0651e53bd09f1
 
-   - SHA: ...
+   - SHA: 0060819964edb18ae542cb6a04eb5fee3e8bd961
 
-   - SHA256: ...
+   - SHA256: ef98626610d1c9b409da6529970a584cf88f0d1e9f3e902a569eddc1332d7864
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Nov 12 08:35:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:35:50 -0800 (PST)
Subject: [pypy-commit] cffi default: hg merge release-1.9
Message-ID: <58271ab6.c515c20a.3e97c.5f50@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2815:87d5f13618fd
Date: 2016-11-12 14:35 +0100
http://bitbucket.org/cffi/cffi/changeset/87d5f13618fd/

Log:	hg merge release-1.9

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.9.0.tar.gz
 
-   - MD5: ...
+   - MD5: 3806f3b4e093fa84fda0651e53bd09f1
 
-   - SHA: ...
+   - SHA: 0060819964edb18ae542cb6a04eb5fee3e8bd961
 
-   - SHA256: ...
+   - SHA256: ef98626610d1c9b409da6529970a584cf88f0d1e9f3e902a569eddc1332d7864
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Nov 12 08:48:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:48:19 -0800 (PST)
Subject: [pypy-commit] cffi release-1.9: hg merge default
Message-ID: <58271da3.0777c20a.ec165.d3c1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.9
Changeset: r2817:b2c90a3b1234
Date: 2016-11-12 14:45 +0100
http://bitbucket.org/cffi/cffi/changeset/b2c90a3b1234/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.9.0"
+#define CFFI_VERSION  "1.9.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
@@ -2153,6 +2153,7 @@
 static PyObject *
 cdata_slice(CDataObject *cd, PySliceObject *slice)
 {
+    char *cdata;
     Py_ssize_t bounds[2];
     CTypeDescrObject *ct = _cdata_getslicearg(cd, slice, bounds);
     if (ct == NULL)
@@ -2165,7 +2166,7 @@
     }
     ct = (CTypeDescrObject *)ct->ct_stuff;
 
-    char *cdata = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0];
+    cdata = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0];
     return new_sized_cdata(cdata, ct, bounds[1]);
 }
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.9.0"
-__version_info__ = (1, 9, 0)
+__version__ = "1.9.1"
+__version_info__ = (1, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.0"
+                               "\ncompiled with cffi version: 1.9.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.9.0'
+release = '1.9.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.1.tar.gz
 
-   - MD5: 3806f3b4e093fa84fda0651e53bd09f1
+   - MD5: ...
 
-   - SHA: 0060819964edb18ae542cb6a04eb5fee3e8bd961
+   - SHA: ...
 
-   - SHA256: ef98626610d1c9b409da6529970a584cf88f0d1e9f3e902a569eddc1332d7864
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -153,7 +153,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.9.0',
+        version='1.9.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Sat Nov 12 08:48:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:48:21 -0800 (PST)
Subject: [pypy-commit] cffi release-1.9: md5/sha
Message-ID: <58271da5.ca73c20a.522f4.3c5e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.9
Changeset: r2818:457833686a4f
Date: 2016-11-12 14:47 +0100
http://bitbucket.org/cffi/cffi/changeset/457833686a4f/

Log:	md5/sha

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.9.1.tar.gz
 
-   - MD5: ...
+   - MD5: b8fa7ccb87790531db3316ab17aa8244
 
-   - SHA: ...
+   - SHA: 16265a4b305d433fb9089b19278502e904b0cb43
 
-   - SHA256: ...
+   - SHA256: 563e0bd53fda03c151573217b3a49b3abad8813de9dd0632e10090f6190fdaf8
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Nov 12 08:48:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:48:22 -0800 (PST)
Subject: [pypy-commit] cffi default: hg merge release-1.9
Message-ID: <58271da6.058a1c0a.3d288.809e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2819:e0ccab2917b5
Date: 2016-11-12 14:47 +0100
http://bitbucket.org/cffi/cffi/changeset/e0ccab2917b5/

Log:	hg merge release-1.9

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,11 +53,11 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.9.1.tar.gz
 
-   - MD5: ...
+   - MD5: b8fa7ccb87790531db3316ab17aa8244
 
-   - SHA: ...
+   - SHA: 16265a4b305d433fb9089b19278502e904b0cb43
 
-   - SHA256: ...
+   - SHA256: 563e0bd53fda03c151573217b3a49b3abad8813de9dd0632e10090f6190fdaf8
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Nov 12 08:48:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 05:48:17 -0800 (PST)
Subject: [pypy-commit] cffi default: update to 1.9.1
Message-ID: <58271da1.44941c0a.9f413.7e92@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2816:64f5c78b61aa
Date: 2016-11-12 14:44 +0100
http://bitbucket.org/cffi/cffi/changeset/64f5c78b61aa/

Log:	update to 1.9.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.9.0"
+#define CFFI_VERSION  "1.9.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.9.0"
-__version_info__ = (1, 9, 0)
+__version__ = "1.9.1"
+__version_info__ = (1, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.0"
+                               "\ncompiled with cffi version: 1.9.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.9.0'
+release = '1.9.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.9.1.tar.gz
 
-   - MD5: 3806f3b4e093fa84fda0651e53bd09f1
+   - MD5: ...
 
-   - SHA: 0060819964edb18ae542cb6a04eb5fee3e8bd961
+   - SHA: ...
 
-   - SHA256: ef98626610d1c9b409da6529970a584cf88f0d1e9f3e902a569eddc1332d7864
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -153,7 +153,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.9.0',
+        version='1.9.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Sat Nov 12 11:08:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 08:08:06 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update www.pypy.org
Message-ID: <58273e66.0777c20a.ec165.fed9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r817:6a3d801ca30d
Date: 2016-11-12 18:07 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6a3d801ca30d/

Log:	update www.pypy.org

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -69,7 +69,7 @@
 <div>
 <div id="main">
 <h1 class="title">Python compatibility</h1>
-<p>PyPy implements the Python language version 2.7.10. It supports all of the core
+<p>PyPy implements the Python language version 2.7.12. It supports all of the core
 language, passing Python test suite (with minor modifications that were
 already accepted in the main python in newer versions). It supports most
 of the commonly used Python <a class="reference external" href="http://docs.python.org/library/">standard library modules</a>; details below.</p>
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,8 +74,8 @@
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.4.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html">what's new in PyPy2.7?</a>
-and <a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.1.html">bugfix release</a>)</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">what's new in PyPy2.7?</a>
+and <a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">bugfix release</a>)</li>
 <li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.5-alpha</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html">what's new in PyPy3.3?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
@@ -115,22 +115,22 @@
 <li>or <a class="reference internal" href="#translate">translate</a> your own PyPy.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-5-4-1">
-<h1>Python2.7 compatible PyPy 5.4.1</h1>
+<div class="section" id="python2-7-compatible-pypy-5-6-0">
+<h1>Python2.7 compatible PyPy 5.6.0</h1>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
@@ -199,7 +199,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in <tt class="docutils literal">/opt</tt>, and
 if you want, put a symlink from somewhere like
-<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.4.1/bin/pypy</span></tt>.  Do
+<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.6.0/bin/pypy</span></tt>.  Do
 not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree &ndash; put
 a symlink to it, otherwise it will not find its libraries.</p>
 </div>
@@ -270,7 +270,7 @@
 <li><p class="first">Get the source code.  The following packages contain the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2">pypy2-v5.4.1-src.tar.bz2</a> (sources)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2">pypy2-v5.6.0-src.tar.bz2</a> (sources)</li>
 </ul>
 <p>Or you can checkout the current trunk using <a class="reference external" href="https://www.mercurial-scm.org/">Mercurial</a> (the trunk
 usually works and is of course more up-to-date):</p>
@@ -410,20 +410,18 @@
 <div class="section" id="checksums">
 <h1>Checksums</h1>
 <p>Here are the checksums for each of the downloads</p>
-<p>pypy2.7-v5.4.1 md5:</p>
+<p>pypy2.7-v5.6.0 md5:</p>
 <pre class="literal-block">
-425ffedf0db4dd737d450aa064ae0e7a  pypy2-v5.4.1-linux-armel.tar.bz2
-15f41409cbadbde3ef22ee60ded8579a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-5940ea0c1077e3bc2ba461bf55800abb  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-23cbad540abff48ea67f5b75368344be  pypy2-v5.4.1-linux32.tar.bz2
-2f4a82ae306f3d30d486fb945ca9c6be  pypy2-v5.4.1-linux64.tar.bz2
-a18cbd990c97f75a08235a35e546a637  pypy2-v5.4.1-osx64.tar.bz2
-90b0650dda1b5bf33f6119f20ba9c3b6  pypy2-v5.4.1-ppc64.tar.bz2
-9a2c1b3124151b79e68f36eb2d4891d1  pypy2-v5.4.1-ppc64le.tar.bz2
-826d4b48e43c84a50dc7e2adc2cb69d4  pypy2-v5.4.1-s390x.tar.bz2
-d1d197d16331aa23a7fd4b5d4c3c1717  pypy2-v5.4.1-src.tar.bz2
-1aab9fe6e7c03e959cde466819034bab  pypy2-v5.4.1-src.zip
-b04aad943aac92862a73b1fd90157a00  pypy2-v5.4.1-win32.zip
+f804f1ee631125aa0a3c8eb8a9f09f20  pypy2-v5.6.0-linux32.tar.bz2
+4d7514f43c5e92e2d066480db86d1932  pypy2-v5.6.0-linux64.tar.bz2
+112b340c407f406dbf79ce715f111467  pypy2-v5.6.0-linux-armel.tar.bz2
+736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
+f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
+c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
+d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
+3579eeea2caa74accf5d35a8cb406a83  pypy2-v5.6.0-win32.zip
 </pre>
 <p>pypy3.3-v5.5.0-alpha md5:</p>
 <pre class="literal-block">
@@ -441,20 +439,18 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 </pre>
-<p>pypy2.7-5.4.1 sha1:</p>
+<p>pypy2.7-5.6.0 sha1:</p>
 <pre class="literal-block">
-a54b2a8b6def85663b10fd956d51fbd052954b83  pypy2-v5.4.1-linux-armel.tar.bz2
-61c9a5269d6d414c4a1d7c41bbc6a45da318f138  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-ea48ffe40887e25adcf1969a6b0e25dbe42a2457  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-bf0ac4668323abead04dd0948a2e541a158e4a00  pypy2-v5.4.1-linux32.tar.bz2
-d16601b04987381c4922a19dacb0ca2591a3b566  pypy2-v5.4.1-linux64.tar.bz2
-4084b7db8ee16a52e7ecb89d10765d961864c67c  pypy2-v5.4.1-osx64.tar.bz2
-98eac6f32f4fe8fce6eb0f0594163f3e01dccced  pypy2-v5.4.1-ppc64.tar.bz2
-fc676af00b356ae48602b531675774f2badfc4fd  pypy2-v5.4.1-ppc64le.tar.bz2
-70b736f1fdb92ae9916630b7cc8954ed92490f64  pypy2-v5.4.1-s390x.tar.bz2
-1143948f50b1b95b55465b246639b4c48251e38e  pypy2-v5.4.1-src.tar.bz2
-a6a9ce0defb401d8777d8af5949d23393416a390  pypy2-v5.4.1-src.zip
-49c2ad75f1531730f7ee466d833d318333915ce0  pypy2-v5.4.1-win32.zip
+78124bd560ab5bd4902a4f55e776c917d851736c  pypy2-v5.6.0-linux32.tar.bz2
+6a5a254ceeb92b0bb9c2e3bfdcaf5a0d9f34df83  pypy2-v5.6.0-linux64.tar.bz2
+8b452ae867f0a5131427386263abe458bf8db845  pypy2-v5.6.0-linux-armel.tar.bz2
+190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
+35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
+ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
+bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
+a7d5d5287c84df3cbbebadee3c455cd4f88528d6  pypy2-v5.6.0-win32.zip
 </pre>
 <p>pypy3.3-v5.5.0-alpha sha1:</p>
 <pre class="literal-block">
@@ -467,20 +463,18 @@
 9eb6aad2d41f7db16c19ae6e438c8f195b0d44fc  pypy3.3-v5.5.0-alpha-src.tar.bz2
 b670cdc685c323b11bfdca2a15f12c5953e6d8b4  pypy3.3-v5.5.0-alpha-src.zip
 </pre>
-<p>pypy2.7-5.4.1 sha256:</p>
+<p>pypy2.7-5.6.0 sha256:</p>
 <pre class="literal-block">
-a1eb5f672aae62606176305e52a51b060ba974b6181ebefcd2c555ecf5f8614f  pypy2-v5.4.1-linux-armel.tar.bz2
-2c4befc4517adec874155a8b6fa0b9d18388943d4ffe778002072db7783e417a  pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-b38646519ee1a888c68f8f4713c122867b4b36693c8acabb38eb827a9d2d51f9  pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-6d1e2386ec1e05dffed493aa2d5e6db5cf5de18d7350d44b85f2e45aa5c9a774  pypy2-v5.4.1-linux32.tar.bz2
-9c85319778224d7fb0c348f55fe3fada15bb579c5f3870a13ad63b42a737dd72  pypy2-v5.4.1-linux64.tar.bz2
-ae9329c8f0a6df431c6224c27c634f998688ac803e8d100cee9a774e6bba38b5  pypy2-v5.4.1-osx64.tar.bz2
-ff118f2dc2393c821595fa7872e4d7bdae2a9a16a854759e103608df383d765a  pypy2-v5.4.1-ppc64.tar.bz2
-684e862cdb281a22c95d73012f7d2693be9fd9cdd171784256da1514ae1c8164  pypy2-v5.4.1-ppc64le.tar.bz2
-4004c86bf2dfdd1d92120f514d296af7602e0c5e2c6526a14ff8b5508c0fe8f7  pypy2-v5.4.1-s390x.tar.bz2
-45dbc50c81498f6f1067201b8fc887074b43b84ee32cc47f15e7db17571e9352  pypy2-v5.4.1-src.tar.bz2
-54b23c11a92dd6328a58787c3d719189f0aa24c872b479acf50094dfb4be0a46  pypy2-v5.4.1-src.zip
-ec729218a820bc2aa2cf1fcacf9de0fee9e04144fe138596198a6b4615505e03  pypy2-v5.4.1-win32.zip
+5d4ad43aed5c5e147f7e7c84766c729f34f63b714b6d224e912a2bb42dc95d62  pypy2-v5.6.0-linux32.tar.bz2
+aad55328cb0673a60b2633dcc3c36cf452917ac906b577eb3aed5876a7666fca  pypy2-v5.6.0-linux64.tar.bz2
+2d1c7820f6368c92bb43a153d2c995f70aa183ff8f1df6916b0d2e57838d8a30  pypy2-v5.6.0-linux-armel.tar.bz2
+2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
+534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
+7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
+f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip
+bab4fa37ef2d32660e291393d955a4e951d5e883abea8bee83be1ec044ddcaac  pypy2-v5.6.0-win32.zip
 </pre>
 <p>pypy3.3-v5.5.0-alpha sha256:</p>
 <pre class="literal-block">
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -72,7 +72,7 @@
 <p>PyPy is a replacement for CPython.  It is built using the RPython
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).</p>
-<p><strong>PyPy</strong> implements <strong>Python 2.7.10</strong>.
+<p><strong>PyPy</strong> implements <strong>Python 2.7.12</strong>.
 It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python
 in newer versions). It supports most of the commonly used Python
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -70,7 +70,7 @@
 <div id="main">
 <h1 class="title">Welcome to PyPy</h1>
 <p>PyPy is a <a class="reference external" href="http://speed.pypy.org/">fast</a>, <a class="reference external" href="compat.html">compliant</a> alternative implementation of the <a class="reference external" href="http://python.org/">Python</a>
-language (2.7.10 and 3.3.5). It has several advantages and distinct features:</p>
+language (2.7.12 and 3.3.5). It has several advantages and distinct features:</p>
 <blockquote>
 <ul>
 <li><p class="first"><strong>Speed:</strong> thanks to its Just-in-Time compiler, Python programs
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,14 +14,14 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v5.4.1** &#8212; (`what's new in PyPy2.7?`_
+* the Python2.7 compatible release &#8212; **PyPy2.7 v5.6.0** &#8212; (`what's new in PyPy2.7?`_
   and `bugfix release`_)
 * the Python3.3 compatible release &#8212; **PyPy3.3 v5.5-alpha** &#8212; (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.0.html
-.. _bugfix release: http://doc.pypy.org/en/latest/release-pypy2.7-v5.4.1.html
+.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html
+.. _bugfix release: http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html
 
 
@@ -77,8 +77,8 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 5.4.1
------------------------------------
+Python2.7 compatible PyPy 5.6.0
+-------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -96,18 +96,18 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
@@ -208,7 +208,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.4.1/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.6.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -302,9 +302,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.4.1-src.tar.bz2`__ (sources)
+   * `pypy2-v5.6.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.4.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::

From pypy.commits at gmail.com  Sat Nov 12 11:14:00 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 08:14:00 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: no bugfix yet
Message-ID: <58273fc8.28c9c20a.cf743.75de@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r818:674a668ca389
Date: 2016-11-12 18:13 +0200
http://bitbucket.org/pypy/pypy.org/changeset/674a668ca389/

Log:	no bugfix yet

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,8 +74,7 @@
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">what's new in PyPy2.7?</a>
-and <a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">bugfix release</a>)</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">what's new in PyPy2.7?</a>)</li>
 <li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.5-alpha</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html">what's new in PyPy3.3?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,14 +14,13 @@
 
 We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v5.6.0** &#8212; (`what's new in PyPy2.7?`_
-  and `bugfix release`_)
+* the Python2.7 compatible release &#8212; **PyPy2.7 v5.6.0** &#8212; (`what's new in PyPy2.7?`_)
+
 * the Python3.3 compatible release &#8212; **PyPy3.3 v5.5-alpha** &#8212; (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
 .. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html
-.. _bugfix release: http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html
 .. _what's new in PyPy3.3?: http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html
 
 

From pypy.commits at gmail.com  Sat Nov 12 12:07:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 Nov 2016 09:07:34 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Add checksums for ppc64
Message-ID: <58274c56.c3181c0a.91181.fc7a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r819:68ca6f396f1a
Date: 2016-11-12 18:07 +0100
http://bitbucket.org/pypy/pypy.org/changeset/68ca6f396f1a/

Log:	Add checksums for ppc64

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -417,6 +417,8 @@
 736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
 65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
 2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
+6aaf25bf60e7a37a2a920eaa7b0e35c5  pypy2-v5.6.0-ppc64.tar.bz2
+57557db44b623047062e2fbd0628dfed  pypy2-v5.6.0-ppc64le.tar.bz2
 f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
 c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
 d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
@@ -446,6 +448,8 @@
 190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
 0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
 5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
+94f4cd90a98425e30dfccab8f9704328da25a6a2  pypy2-v5.6.0-ppc64.tar.bz2
+d3e39c88d8711b49c6584508c6966a80d5cb772c  pypy2-v5.6.0-ppc64le.tar.bz2
 35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
 ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
 bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
@@ -470,6 +474,8 @@
 2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
 0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
 6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
+51bbe172b7b631fa48fab6a2bd2b5411f99a87cb7ebb47ed0a29affd147667c6  pypy2-v5.6.0-ppc64.tar.bz2
+81a82bcb87cb95101e6019a404eca956e4f18a685f588ae88720026641c40d0f  pypy2-v5.6.0-ppc64le.tar.bz2
 534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
 7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
 f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -466,6 +466,8 @@
     736bd4fb085959394c97df5e40bec6e6  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
     65beda937953e697f212c9acfde91e8c  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
     2d59a3fb9abdc224193b4206814fb5ad  pypy2-v5.6.0-osx64.tar.bz2
+    6aaf25bf60e7a37a2a920eaa7b0e35c5  pypy2-v5.6.0-ppc64.tar.bz2
+    57557db44b623047062e2fbd0628dfed  pypy2-v5.6.0-ppc64le.tar.bz2
     f027e7818a1c4a8ad14259f8bc6cbeec  pypy2-v5.6.0-s390x.tar.bz2
     c3fc7187061fec762269496f1f5daa86  pypy2-v5.6.0-src.tar.bz2
     d5c3a0cce5f07c739b9f6e63b55247e8  pypy2-v5.6.0-src.zip
@@ -495,6 +497,8 @@
     190295288f9b6f938b112109f6594235a14f1df3  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
     0004ccae2872b106b46f479a94c43c8dd7811eba  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
     5021b044ddf68155ca1bb2b41acc5b41d44dd2db  pypy2-v5.6.0-osx64.tar.bz2
+    94f4cd90a98425e30dfccab8f9704328da25a6a2  pypy2-v5.6.0-ppc64.tar.bz2
+    d3e39c88d8711b49c6584508c6966a80d5cb772c  pypy2-v5.6.0-ppc64le.tar.bz2
     35056ff8fba233ecd896da226bf0949b83dfb2ca  pypy2-v5.6.0-s390x.tar.bz2
     ecfd7efdee340ee170db56a79c18bde3e37cd326  pypy2-v5.6.0-src.tar.bz2
     bf805de05a11bda6182efd7f14c3b6b1945a6f9a  pypy2-v5.6.0-src.zip
@@ -519,6 +523,8 @@
     2c430240cecb562102c677598f106aa57899f00cd37f719989e18ed9ca44bd01  pypy2-v5.6.0-linux-armhf-raring.tar.bz2
     0f69c40a38d72254bf12094620bda9d2face758f763cd0d989588642d81eccae  pypy2-v5.6.0-linux-armhf-raspbian.tar.bz2
     6121f791f440564d3a0a41315e9f2d2d61bc484654acffd85d9e1c6e92b85c36  pypy2-v5.6.0-osx64.tar.bz2
+    51bbe172b7b631fa48fab6a2bd2b5411f99a87cb7ebb47ed0a29affd147667c6  pypy2-v5.6.0-ppc64.tar.bz2
+    81a82bcb87cb95101e6019a404eca956e4f18a685f588ae88720026641c40d0f  pypy2-v5.6.0-ppc64le.tar.bz2
     534284c5a35da647f623d1937f99ae720927de8b4b1ce20143d860050f644376  pypy2-v5.6.0-s390x.tar.bz2
     7411448045f77eb9e087afdce66fe7eafda1876c9e17aad88cf891f762b608b0  pypy2-v5.6.0-src.tar.bz2
     f9f86ac8267d9997c314d1345d09a20a4528352285ed5622e35cf6e79480fe28  pypy2-v5.6.0-src.zip

From pypy.commits at gmail.com  Sat Nov 12 13:36:13 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 10:36:13 -0800 (PST)
Subject: [pypy-commit] pypy default: add links to RPython projects,
 needs updating
Message-ID: <5827611d.44e61c0a.dbd1e.0259@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88331:1d0bf0df0fb8
Date: 2016-11-12 20:33 +0200
http://bitbucket.org/pypy/pypy/changeset/1d0bf0df0fb8/

Log:	add links to RPython projects, needs updating

diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/examples.rst
@@ -0,0 +1,37 @@
+Projects Using RPython
+======================
+
+A very time-dependent list of interpreters written in RPython. Corrections welcome,
+this list was last curated in
+September 2015
+
+Actively Developed:
+
+  * PyPy, Python, very complete and maintained, http://pypy.org
+  * Converge 2, complete, maintained, http://convergepl.org/
+  * Pydgin, CPU emulation framework, supports ARM well, jitted, active development, https://github.com/cornell-brg/pydgin
+  * Pyrolog, Prolog, core complete, extensions missing, sort of maintained, http://bitbucket.org/cfbolz/pyrolog
+  * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting there, in active development https://github.com/HPI-SWA-Lab/RSqueak
+  * Pixie, 'A small, fast, native lisp with "magical" powers', jitted, maintained, https://github.com/pixie-lang/pixie
+  * Monte, 'A dynamic language inspired by Python and E.' has an rpython implementation, in active development, https://github.com/monte-language/typhon
+  * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
+  * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
+
+Inactive?:
+
+  * Topaz, Ruby, major functionality complete, library missing, inactive http://topazruby.com
+  * Rapydo, R, execution semantics complete, most builtins missing, inactive, http://bitbucket.org/cfbolz/rapydo
+  * Hippy, PHP, proof of concept, inactive, http://morepypy.blogspot.de/2012/07/hello-everyone.html
+  * Scheme, no clue about completeness, inactive, http://bitbucket.org/pypy/lang-scheme/
+  * PyGirl, Gameboy emulator, works but there is a bug somewhere, does not use JIT, unmaintained, http://bitbucket.org/pypy/lang-gameboy
+  * Javascript, proof of concept, inactive, http://bitbucket.org/pypy/lang-js
+  * An implementation of Notch's DCPU-16, https://github.com/AlekSi/dcpu16py/tree/pypy-again
+  * Haskell, core of the language works, but not many libraries, inactive http://bitbucket.org/cfbolz/haskell-python
+  * IO, no clue about completeness, inactive https://bitbucket.org/pypy/lang-io
+  * Qoppy, an implementation Qoppa, which is a scheme without special forms: https://github.com/timfel/qoppy
+  * XlispX, a toy Lisp: https://bitbucket.org/rxe/xlispx
+  * RPySOM, an RPython implementation of SOM (Simple Object Model) https://github.com/SOM-st/RPySOM          
+  * SQPyte, really experimental implementation of the SQLite bytecode VM, jitted, probably inactive, https://bitbucket.org/softdevteam/sqpyte
+  * Icbink, an implementation of Kernel, core complete, naive, no JIT optimizations yet, on hiatus https://github.com/euccastro/icbink
+  * PyPy.js, adds a JavaScript backend to pypy, http://pypyjs.org
+
diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -25,7 +25,6 @@
    architecture
    faq
 
-
 User Documentation
 ------------------
 
@@ -48,6 +47,7 @@
    rpython
    rlib
    rffi
+   examples
 
 
 RPython internals

From pypy.commits at gmail.com  Sat Nov 12 15:24:48 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 12:24:48 -0800 (PST)
Subject: [pypy-commit] pypy default: first cut at updating,
 glanced at "Actively Developed" to make sure they actually are
Message-ID: <58277a90.233ac20a.6de12.ca0b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88332:f44b35fec7fc
Date: 2016-11-12 22:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f44b35fec7fc/

Log:	first cut at updating, glanced at "Actively Developed" to make sure
	they actually are

diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
--- a/rpython/doc/examples.rst
+++ b/rpython/doc/examples.rst
@@ -3,21 +3,25 @@
 
 A very time-dependent list of interpreters written in RPython. Corrections welcome,
 this list was last curated in
-September 2015
+Nov 2016
 
 Actively Developed:
 
   * PyPy, Python, very complete and maintained, http://pypy.org
-  * Converge 2, complete, maintained, http://convergepl.org/
   * Pydgin, CPU emulation framework, supports ARM well, jitted, active development, https://github.com/cornell-brg/pydgin
-  * Pyrolog, Prolog, core complete, extensions missing, sort of maintained, http://bitbucket.org/cfbolz/pyrolog
   * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting there, in active development https://github.com/HPI-SWA-Lab/RSqueak
   * Pixie, 'A small, fast, native lisp with "magical" powers', jitted, maintained, https://github.com/pixie-lang/pixie
   * Monte, 'A dynamic language inspired by Python and E.' has an rpython implementation, in active development, https://github.com/monte-language/typhon
   * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
   * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
 
-Inactive?:
+Complete, functioning, but inactive
+
+  * Converge 2, complete, last release version 2.1 in Feb 2015, http://convergepl.org/
+  * Pyrolog, Prolog, core complete, extensions missing, last commit in Nov 2015, http://bitbucket.org/cfbolz/pyrolog
+  * PyPy.js, adds a JavaScript backend to pypy, http://pypyjs.org
+
+Inactive (last reviewed Sept 2015):
 
   * Topaz, Ruby, major functionality complete, library missing, inactive http://topazruby.com
   * Rapydo, R, execution semantics complete, most builtins missing, inactive, http://bitbucket.org/cfbolz/rapydo
@@ -33,5 +37,4 @@
   * RPySOM, an RPython implementation of SOM (Simple Object Model) https://github.com/SOM-st/RPySOM          
   * SQPyte, really experimental implementation of the SQLite bytecode VM, jitted, probably inactive, https://bitbucket.org/softdevteam/sqpyte
   * Icbink, an implementation of Kernel, core complete, naive, no JIT optimizations yet, on hiatus https://github.com/euccastro/icbink
-  * PyPy.js, adds a JavaScript backend to pypy, http://pypyjs.org
 

From pypy.commits at gmail.com  Sat Nov 12 15:24:50 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 12:24:50 -0800 (PST)
Subject: [pypy-commit] pypy default: point to list of projects using rpython
Message-ID: <58277a92.4a6ec20a.f675a.d0aa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88333:111df2d6caac
Date: 2016-11-12 22:24 +0200
http://bitbucket.org/pypy/pypy/changeset/111df2d6caac/

Log:	point to list of projects using rpython

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -72,7 +72,7 @@
   * **s390x** running Linux
 
 .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
-.. _`dynamic languages`: http://pypyjs.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
 
 Other Highlights (since 5.4 released Aug 31, 2016)
 =========================================================

From pypy.commits at gmail.com  Sat Nov 12 15:38:57 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 12:38:57 -0800 (PST)
Subject: [pypy-commit] pypy default: add lever,
 typhon (Samureus on IRC plus my interpretation of the README)
Message-ID: <58277de1.c5311c0a.74061.13e3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88334:2e5362135372
Date: 2016-11-12 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/2e5362135372/

Log:	add lever, typhon (Samureus on IRC plus my interpretation of the
	README)

diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
--- a/rpython/doc/examples.rst
+++ b/rpython/doc/examples.rst
@@ -12,8 +12,10 @@
   * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting there, in active development https://github.com/HPI-SWA-Lab/RSqueak
   * Pixie, 'A small, fast, native lisp with "magical" powers', jitted, maintained, https://github.com/pixie-lang/pixie
   * Monte, 'A dynamic language inspired by Python and E.' has an rpython implementation, in active development, https://github.com/monte-language/typhon
+  * Typhon, 'A virtual machine for Monte', in active development, https://github.com/monte-language/typhon
   * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
   * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
+  * Lever, a dynamic language with a modifiable grammer, actively developed, https://github.com/cheery/lever
 
 Complete, functioning, but inactive
 

From pypy.commits at gmail.com  Sat Nov 12 15:46:51 2016
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 Nov 2016 12:46:51 -0800 (PST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <58277fbb.e5bfc20a.a6216.d977@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88335:f563b743a65c
Date: 2016-11-12 22:46 +0200
http://bitbucket.org/pypy/pypy/changeset/f563b743a65c/

Log:	typo

diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
--- a/rpython/doc/examples.rst
+++ b/rpython/doc/examples.rst
@@ -15,7 +15,7 @@
   * Typhon, 'A virtual machine for Monte', in active development, https://github.com/monte-language/typhon
   * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
   * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
-  * Lever, a dynamic language with a modifiable grammer, actively developed, https://github.com/cheery/lever
+  * Lever, a dynamic language with a modifiable grammar, actively developed, https://github.com/cheery/lever
 
 Complete, functioning, but inactive
 

From pypy.commits at gmail.com  Sun Nov 13 04:21:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 01:21:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Add an official alternative format for
 debug_merge_point, for
Message-ID: <582830a6.c4811c0a.87ceb.baf8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88336:66353dacc7ad
Date: 2016-11-13 10:20 +0100
http://bitbucket.org/pypy/pypy/changeset/66353dacc7ad/

Log:	Add an official alternative format for debug_merge_point, for all
	interpreters != PyPy2

diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -50,6 +50,9 @@
             current_lineno = opcode.lineno
         self.source = source.split("\n")
 
+    def get_opcode_from_info(self, info):
+        return self.map[info.bytecode_no]
+
 def _setup():
     for opcode in opname:
         if not opcode.startswith('<'):
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -11,15 +11,28 @@
     filename = None
     bytecode_no = 0
     bytecode_name = None
-    m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
-                  arg)
+    mask = 0
+    # generic format: the numbers are 'startlineno-currentlineno',
+    # and this function returns currentlineno as the value
+    # 'bytecode_no = currentlineno ^ -1': i.e. it abuses bytecode_no,
+    # which doesn't make sense in the generic format, as a negative
+    # number
+    m = re.match(r'(.+?);(.+?):(\d+)-(\d+) (.*)', arg)
+    if m is not None:
+        mask = -1
+    else:
+        # PyPy2 format: bytecode_no is really a bytecode index,
+        # which must be turned into a real line number by parsing the
+        # source file
+        m = re.search(r'<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] '
+                      r'line (\d+)> #(\d+) (\w+)', arg)
     if m is None:
         # a non-code loop, like StrLiteralSearch or something
         if arg:
             bytecode_name = arg
     else:
         name, filename, lineno, bytecode_no, bytecode_name = m.groups()
-    return name, bytecode_name, filename, int(lineno), int(bytecode_no)
+    return name, bytecode_name, filename, int(lineno), int(bytecode_no) ^ mask
 
 class Op(object):
     bridge = None
@@ -195,8 +208,9 @@
              self.startlineno, self.bytecode_no) = parsed
         self.operations = operations
         self.storage = storage
+        generic_format = (self.bytecode_no < 0)
         self.code = storage.disassemble_code(self.filename, self.startlineno,
-                                             self.name)
+                                             self.name, generic_format)
 
     def repr(self):
         if self.filename is None:
@@ -213,7 +227,7 @@
     def getopcode(self):
         if self.code is None:
             return None
-        return self.code.map[self.bytecode_no]
+        return self.code.get_opcode_from_info(self)
 
     def getlineno(self):
         code = self.getopcode()
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -5,6 +5,7 @@
 
 import py
 import os
+import linecache
 from rpython.tool.disassembler import dis
 from rpython.tool.jitlogparser.module_finder import gather_all_code_objs
 
@@ -29,7 +30,10 @@
             self.codes[fname] = res
             return res
 
-    def disassemble_code(self, fname, startlineno, name):
+    def disassemble_code(self, fname, startlineno, name, generic_format=False):
+        # 'generic_format' is False for PyPy2 (returns a
+        # disassembler.CodeRepresentation) or True otherwise (returns a
+        # GenericCode, without attempting any disassembly)
         try:
             if py.path.local(fname).check(file=False):
                 return None # cannot find source file
@@ -39,6 +43,10 @@
         try:
             return self.disassembled_codes[key]
         except KeyError:
+            pass
+        if generic_format:
+            res = GenericCode(fname, startlineno, name)
+        else:
             codeobjs = self.load_code(fname)
             if (startlineno, name) not in codeobjs:
                 # cannot find the code obj at this line: this can happen for
@@ -50,8 +58,8 @@
                 return None
             code = codeobjs[(startlineno, name)]
             res = dis(code)
-            self.disassembled_codes[key] = res
-            return res
+        self.disassembled_codes[key] = res
+        return res
 
     def reconnect_loops(self, loops):
         """ Re-connect loops in a way that entry bridges are filtered out
@@ -80,3 +88,40 @@
             res.append(loop)
         self.loops = res
         return res
+
+
+class GenericCode(object):
+    def __init__(self, fname, startlineno, name):
+        self._fname = fname
+        self._startlineno = startlineno
+        self._name = name
+        self._first_bytecodes = {}     # {lineno: info}
+        self._source = None
+
+    def __repr__(self):
+        return 'GenericCode(%r, %r, %r)' % (
+            self._fname, self._startlineno, self._name)
+
+    def get_opcode_from_info(self, info):
+        lineno = ~info.bytecode_no
+        if self._first_bytecodes.setdefault(lineno, info) is info:
+            # this is the first opcode of the line---or, at least,
+            # the first time we ask for an Opcode on that line.
+            line_starts_here = True
+        else:
+            line_starts_here = False
+        return GenericOpcode(lineno, line_starts_here)
+
+    @property
+    def source(self):
+        if self._source is None:
+            src = linecache.getlines(self._fname)
+            if self._startlineno > 0:
+                src = src[self._startlineno - 1:]
+            self._source = [s.rstrip('\n\r') for s in src]
+        return self._source
+
+class GenericOpcode(object):
+    def __init__(self, lineno, line_starts_here):
+        self.lineno = lineno
+        self.line_starts_here = line_starts_here
diff --git a/rpython/tool/jitlogparser/test/test_parser.py b/rpython/tool/jitlogparser/test/test_parser.py
--- a/rpython/tool/jitlogparser/test/test_parser.py
+++ b/rpython/tool/jitlogparser/test/test_parser.py
@@ -2,7 +2,8 @@
                                               Function, adjust_bridges,
                                               import_log, split_trace, Op,
                                               parse_log_counts)
-from rpython.tool.jitlogparser.storage import LoopStorage
+from rpython.tool.jitlogparser.storage import LoopStorage, GenericCode
+from rpython.tool.udir import udir
 import py, sys
 from rpython.jit.backend.detect_cpu import autodetect
 from rpython.jit.backend.tool.viewcode import ObjdumpNotFound
@@ -381,4 +382,58 @@
     """)
     f = Function.from_operations(loop.operations, LoopStorage())
     assert len(f.chunks) == 2
-    
+
+def test_embedded_lineno():
+    # debug_merge_point() can have a text that is either:
+    #
+    # * the PyPy2's  <code object %s. file '%s'. line %d> #%d %s>
+    #                 funcname, filename, lineno, bytecode_no, bytecode_name
+    #
+    # * a standard text of the form  %s;%s:%d-%d-%d %s
+    #           funcname, filename, startlineno, curlineno, endlineno, anything
+    #
+    # * or anything else, which is not specially recognized but shouldn't crash
+    #
+    sourcefile = str(udir.join('test_embedded_lineno.src'))
+    with open(sourcefile, 'w') as f:
+        print >> f, "A#1"
+        print >> f, "B#2"
+        print >> f, "C#3"
+        print >> f, "D#4"
+        print >> f, "E#5"
+        print >> f, "F#6"
+    loop = parse("""
+    []
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2 one')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2 two')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4 three')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4 four')
+    """ % {'filename': sourcefile})
+    f = Function.from_operations(loop.operations, LoopStorage())
+
+    expect = [(2, 'one', True),
+              (2, 'two', False),
+              (4, 'three', True),
+              (4, 'four', False)]
+    assert len(f.chunks) == len(expect)
+
+    code_seen = set()
+    for chunk, (expected_lineno,
+                expected_bytecode_name,
+                expected_line_starts_here) in zip(f.chunks, expect):
+        assert chunk.name == 'myfunc'
+        assert chunk.bytecode_name == expected_bytecode_name
+        assert chunk.filename == sourcefile
+        assert chunk.startlineno == 2
+        assert chunk.bytecode_no == ~expected_lineno     # half-abuse
+        assert chunk.has_valid_code()
+        assert chunk.lineno == expected_lineno
+        assert chunk.line_starts_here == expected_line_starts_here
+        code_seen.add(chunk.code)
+
+    assert len(code_seen) == 1
+    code, = code_seen
+    assert code.source[0] == "B#2"
+    assert code.source[1] == "C#3"
+    assert code.source[4] == "F#6"
+    py.test.raises(IndexError, "code.source[5]")

From pypy.commits at gmail.com  Sun Nov 13 04:45:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 01:45:23 -0800 (PST)
Subject: [pypy-commit] pypy default: Change the space separator to a '~'
 because if we have no bytecode_name
Message-ID: <58283633.876ec20a.9fe9f.7c34@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88337:e17c0d890864
Date: 2016-11-13 10:44 +0100
http://bitbucket.org/pypy/pypy/changeset/e17c0d890864/

Log:	Change the space separator to a '~' because if we have no
	bytecode_name to provide, it's tempting to leave out the final
	space, and then the regexp doesn't match

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -17,7 +17,7 @@
     # 'bytecode_no = currentlineno ^ -1': i.e. it abuses bytecode_no,
     # which doesn't make sense in the generic format, as a negative
     # number
-    m = re.match(r'(.+?);(.+?):(\d+)-(\d+) (.*)', arg)
+    m = re.match(r'(.+?);(.+?):(\d+)-(\d+)~(.*)', arg)
     if m is not None:
         mask = -1
     else:
diff --git a/rpython/tool/jitlogparser/test/test_parser.py b/rpython/tool/jitlogparser/test/test_parser.py
--- a/rpython/tool/jitlogparser/test/test_parser.py
+++ b/rpython/tool/jitlogparser/test/test_parser.py
@@ -404,16 +404,16 @@
         print >> f, "F#6"
     loop = parse("""
     []
-    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2 one')
-    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2 two')
-    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4 three')
-    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4 four')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~one')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~two')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~four')
     """ % {'filename': sourcefile})
     f = Function.from_operations(loop.operations, LoopStorage())
 
     expect = [(2, 'one', True),
               (2, 'two', False),
-              (4, 'three', True),
+              (4, '', True),
               (4, 'four', False)]
     assert len(f.chunks) == len(expect)
 

From pypy.commits at gmail.com  Sun Nov 13 04:48:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 01:48:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582836fc.46831c0a.aa12.9aa4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88338:65a37f0d6b93
Date: 2016-11-13 10:47 +0100
http://bitbucket.org/pypy/pypy/changeset/65a37f0d6b93/

Log:	hg merge default

diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -72,7 +72,7 @@
   * **s390x** running Linux
 
 .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
-.. _`dynamic languages`: http://pypyjs.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
 
 Other Highlights (since 5.4 released Aug 31, 2016)
 =========================================================
diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/examples.rst
@@ -0,0 +1,42 @@
+Projects Using RPython
+======================
+
+A very time-dependent list of interpreters written in RPython. Corrections welcome,
+this list was last curated in
+Nov 2016
+
+Actively Developed:
+
+  * PyPy, Python, very complete and maintained, http://pypy.org
+  * Pydgin, CPU emulation framework, supports ARM well, jitted, active development, https://github.com/cornell-brg/pydgin
+  * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting there, in active development https://github.com/HPI-SWA-Lab/RSqueak
+  * Pixie, 'A small, fast, native lisp with "magical" powers', jitted, maintained, https://github.com/pixie-lang/pixie
+  * Monte, 'A dynamic language inspired by Python and E.' has an rpython implementation, in active development, https://github.com/monte-language/typhon
+  * Typhon, 'A virtual machine for Monte', in active development, https://github.com/monte-language/typhon
+  * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
+  * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
+  * Lever, a dynamic language with a modifiable grammar, actively developed, https://github.com/cheery/lever
+
+Complete, functioning, but inactive
+
+  * Converge 2, complete, last release version 2.1 in Feb 2015, http://convergepl.org/
+  * Pyrolog, Prolog, core complete, extensions missing, last commit in Nov 2015, http://bitbucket.org/cfbolz/pyrolog
+  * PyPy.js, adds a JavaScript backend to pypy, http://pypyjs.org
+
+Inactive (last reviewed Sept 2015):
+
+  * Topaz, Ruby, major functionality complete, library missing, inactive http://topazruby.com
+  * Rapydo, R, execution semantics complete, most builtins missing, inactive, http://bitbucket.org/cfbolz/rapydo
+  * Hippy, PHP, proof of concept, inactive, http://morepypy.blogspot.de/2012/07/hello-everyone.html
+  * Scheme, no clue about completeness, inactive, http://bitbucket.org/pypy/lang-scheme/
+  * PyGirl, Gameboy emulator, works but there is a bug somewhere, does not use JIT, unmaintained, http://bitbucket.org/pypy/lang-gameboy
+  * Javascript, proof of concept, inactive, http://bitbucket.org/pypy/lang-js
+  * An implementation of Notch's DCPU-16, https://github.com/AlekSi/dcpu16py/tree/pypy-again
+  * Haskell, core of the language works, but not many libraries, inactive http://bitbucket.org/cfbolz/haskell-python
+  * IO, no clue about completeness, inactive https://bitbucket.org/pypy/lang-io
+  * Qoppy, an implementation Qoppa, which is a scheme without special forms: https://github.com/timfel/qoppy
+  * XlispX, a toy Lisp: https://bitbucket.org/rxe/xlispx
+  * RPySOM, an RPython implementation of SOM (Simple Object Model) https://github.com/SOM-st/RPySOM          
+  * SQPyte, really experimental implementation of the SQLite bytecode VM, jitted, probably inactive, https://bitbucket.org/softdevteam/sqpyte
+  * Icbink, an implementation of Kernel, core complete, naive, no JIT optimizations yet, on hiatus https://github.com/euccastro/icbink
+
diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -25,7 +25,6 @@
    architecture
    faq
 
-
 User Documentation
 ------------------
 
@@ -48,6 +47,7 @@
    rpython
    rlib
    rffi
+   examples
 
 
 RPython internals
diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -50,6 +50,9 @@
             current_lineno = opcode.lineno
         self.source = source.split("\n")
 
+    def get_opcode_from_info(self, info):
+        return self.map[info.bytecode_no]
+
 def _setup():
     for opcode in opname:
         if not opcode.startswith('<'):
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -11,15 +11,28 @@
     filename = None
     bytecode_no = 0
     bytecode_name = None
-    m = re.search('<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] line (\d+)> #(\d+) (\w+)',
-                  arg)
+    mask = 0
+    # generic format: the numbers are 'startlineno-currentlineno',
+    # and this function returns currentlineno as the value
+    # 'bytecode_no = currentlineno ^ -1': i.e. it abuses bytecode_no,
+    # which doesn't make sense in the generic format, as a negative
+    # number
+    m = re.match(r'(.+?);(.+?):(\d+)-(\d+)~(.*)', arg)
+    if m is not None:
+        mask = -1
+    else:
+        # PyPy2 format: bytecode_no is really a bytecode index,
+        # which must be turned into a real line number by parsing the
+        # source file
+        m = re.search(r'<code object ([<>\w]+)[\.,] file \'(.+?)\'[\.,] '
+                      r'line (\d+)> #(\d+) (\w+)', arg)
     if m is None:
         # a non-code loop, like StrLiteralSearch or something
         if arg:
             bytecode_name = arg
     else:
         name, filename, lineno, bytecode_no, bytecode_name = m.groups()
-    return name, bytecode_name, filename, int(lineno), int(bytecode_no)
+    return name, bytecode_name, filename, int(lineno), int(bytecode_no) ^ mask
 
 class Op(object):
     bridge = None
@@ -195,8 +208,9 @@
              self.startlineno, self.bytecode_no) = parsed
         self.operations = operations
         self.storage = storage
+        generic_format = (self.bytecode_no < 0)
         self.code = storage.disassemble_code(self.filename, self.startlineno,
-                                             self.name)
+                                             self.name, generic_format)
 
     def repr(self):
         if self.filename is None:
@@ -213,7 +227,7 @@
     def getopcode(self):
         if self.code is None:
             return None
-        return self.code.map[self.bytecode_no]
+        return self.code.get_opcode_from_info(self)
 
     def getlineno(self):
         code = self.getopcode()
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -5,6 +5,7 @@
 
 import py
 import os
+import linecache
 from rpython.tool.disassembler import dis
 from rpython.tool.jitlogparser.module_finder import gather_all_code_objs
 
@@ -29,7 +30,10 @@
             self.codes[fname] = res
             return res
 
-    def disassemble_code(self, fname, startlineno, name):
+    def disassemble_code(self, fname, startlineno, name, generic_format=False):
+        # 'generic_format' is False for PyPy2 (returns a
+        # disassembler.CodeRepresentation) or True otherwise (returns a
+        # GenericCode, without attempting any disassembly)
         try:
             if py.path.local(fname).check(file=False):
                 return None # cannot find source file
@@ -39,6 +43,10 @@
         try:
             return self.disassembled_codes[key]
         except KeyError:
+            pass
+        if generic_format:
+            res = GenericCode(fname, startlineno, name)
+        else:
             codeobjs = self.load_code(fname)
             if (startlineno, name) not in codeobjs:
                 # cannot find the code obj at this line: this can happen for
@@ -50,8 +58,8 @@
                 return None
             code = codeobjs[(startlineno, name)]
             res = dis(code)
-            self.disassembled_codes[key] = res
-            return res
+        self.disassembled_codes[key] = res
+        return res
 
     def reconnect_loops(self, loops):
         """ Re-connect loops in a way that entry bridges are filtered out
@@ -80,3 +88,40 @@
             res.append(loop)
         self.loops = res
         return res
+
+
+class GenericCode(object):
+    def __init__(self, fname, startlineno, name):
+        self._fname = fname
+        self._startlineno = startlineno
+        self._name = name
+        self._first_bytecodes = {}     # {lineno: info}
+        self._source = None
+
+    def __repr__(self):
+        return 'GenericCode(%r, %r, %r)' % (
+            self._fname, self._startlineno, self._name)
+
+    def get_opcode_from_info(self, info):
+        lineno = ~info.bytecode_no
+        if self._first_bytecodes.setdefault(lineno, info) is info:
+            # this is the first opcode of the line---or, at least,
+            # the first time we ask for an Opcode on that line.
+            line_starts_here = True
+        else:
+            line_starts_here = False
+        return GenericOpcode(lineno, line_starts_here)
+
+    @property
+    def source(self):
+        if self._source is None:
+            src = linecache.getlines(self._fname)
+            if self._startlineno > 0:
+                src = src[self._startlineno - 1:]
+            self._source = [s.rstrip('\n\r') for s in src]
+        return self._source
+
+class GenericOpcode(object):
+    def __init__(self, lineno, line_starts_here):
+        self.lineno = lineno
+        self.line_starts_here = line_starts_here
diff --git a/rpython/tool/jitlogparser/test/test_parser.py b/rpython/tool/jitlogparser/test/test_parser.py
--- a/rpython/tool/jitlogparser/test/test_parser.py
+++ b/rpython/tool/jitlogparser/test/test_parser.py
@@ -2,7 +2,8 @@
                                               Function, adjust_bridges,
                                               import_log, split_trace, Op,
                                               parse_log_counts)
-from rpython.tool.jitlogparser.storage import LoopStorage
+from rpython.tool.jitlogparser.storage import LoopStorage, GenericCode
+from rpython.tool.udir import udir
 import py, sys
 from rpython.jit.backend.detect_cpu import autodetect
 from rpython.jit.backend.tool.viewcode import ObjdumpNotFound
@@ -381,4 +382,58 @@
     """)
     f = Function.from_operations(loop.operations, LoopStorage())
     assert len(f.chunks) == 2
-    
+
+def test_embedded_lineno():
+    # debug_merge_point() can have a text that is either:
+    #
+    # * the PyPy2's  <code object %s. file '%s'. line %d> #%d %s>
+    #                 funcname, filename, lineno, bytecode_no, bytecode_name
+    #
+    # * a standard text of the form  %s;%s:%d-%d-%d %s
+    #           funcname, filename, startlineno, curlineno, endlineno, anything
+    #
+    # * or anything else, which is not specially recognized but shouldn't crash
+    #
+    sourcefile = str(udir.join('test_embedded_lineno.src'))
+    with open(sourcefile, 'w') as f:
+        print >> f, "A#1"
+        print >> f, "B#2"
+        print >> f, "C#3"
+        print >> f, "D#4"
+        print >> f, "E#5"
+        print >> f, "F#6"
+    loop = parse("""
+    []
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~one')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~two')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~')
+    debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~four')
+    """ % {'filename': sourcefile})
+    f = Function.from_operations(loop.operations, LoopStorage())
+
+    expect = [(2, 'one', True),
+              (2, 'two', False),
+              (4, '', True),
+              (4, 'four', False)]
+    assert len(f.chunks) == len(expect)
+
+    code_seen = set()
+    for chunk, (expected_lineno,
+                expected_bytecode_name,
+                expected_line_starts_here) in zip(f.chunks, expect):
+        assert chunk.name == 'myfunc'
+        assert chunk.bytecode_name == expected_bytecode_name
+        assert chunk.filename == sourcefile
+        assert chunk.startlineno == 2
+        assert chunk.bytecode_no == ~expected_lineno     # half-abuse
+        assert chunk.has_valid_code()
+        assert chunk.lineno == expected_lineno
+        assert chunk.line_starts_here == expected_line_starts_here
+        code_seen.add(chunk.code)
+
+    assert len(code_seen) == 1
+    code, = code_seen
+    assert code.source[0] == "B#2"
+    assert code.source[1] == "C#3"
+    assert code.source[4] == "F#6"
+    py.test.raises(IndexError, "code.source[5]")
diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py
--- a/rpython/translator/c/gcc/trackgcroot.py
+++ b/rpython/translator/c/gcc/trackgcroot.py
@@ -926,6 +926,13 @@
                 assert  lineoffset in (1,2)
                 return [InsnStackAdjust(-4)]
 
+        if target.startswith('__x86.get_pc_thunk.'):
+            # special case, found on x86-32: these static functions
+            # contain only a simple load of some non-GC pointer to
+            # a specific register (not necessarily EAX)
+            reg = '%e' + target.split('.')[-1]
+            return [InsnSetLocal(reg)]
+
         insns = [InsnCall(target, self.currentlineno),
                  InsnSetLocal(self.EAX)]      # the result is there
         if self.format in ('mingw32', 'msvc'):

From pypy.commits at gmail.com  Sun Nov 13 05:40:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 02:40:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: make PyPy3 generate debug_merge_points in
 the new generic format
Message-ID: <58284338.45ad1c0a.ae9d6.0f0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88339:625d50ec6fdb
Date: 2016-11-13 10:50 +0000
http://bitbucket.org/pypy/pypy/changeset/625d50ec6fdb/

Log:	make PyPy3 generate debug_merge_points in the new generic format

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,9 +35,12 @@
 
 def get_printable_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
-    from rpython.rlib.runicode import unicode_encode_utf_8
-    name = opcode_method_names[ord(bytecode.co_code[next_instr])]
-    return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
+    from pypy.interpreter.pytraceback import offset2lineno
+    bytecode_name = opcode_method_names[ord(bytecode.co_code[next_instr])]
+    lineno = offset2lineno(bytecode, intmask(next_instr))
+    return '%s;%s:%d-%d~#%d %s' % (
+        bytecode.co_name, bytecode.co_filename, bytecode.co_firstlineno,
+        lineno, next_instr, bytecode_name)
 
 def get_unique_id(next_instr, is_being_profiled, bytecode):
     from rpython.rlib import rvmprof
@@ -47,7 +50,7 @@
             jl.MP_SCOPE, jl.MP_INDEX, jl.MP_OPCODE)
 def get_location(next_instr, is_being_profiled, bytecode):
     from pypy.tool.stdlib_opcode import opcode_method_names
-    from rpython.tool.error import offset2lineno
+    from pypy.interpreter.pytraceback import offset2lineno
     bcindex = ord(bytecode.co_code[next_instr])
     opname = ""
     if 0 <= bcindex < len(opcode_method_names):

From pypy.commits at gmail.com  Sun Nov 13 08:28:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 05:28:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Starting to pass these tests (many many
 failures)
Message-ID: <58286a98.624fc20a.f7b3f.8eb0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88340:dd2ec76131ce
Date: 2016-11-13 13:38 +0000
http://bitbucket.org/pypy/pypy/changeset/dd2ec76131ce/

Log:	Starting to pass these tests (many many failures)

diff --git a/pypy/module/pypyjit/test/conftest.py b/pypy/module/pypyjit/test_pypy_c/conftest.py
copy from pypy/module/pypyjit/test/conftest.py
copy to pypy/module/pypyjit/test_pypy_c/conftest.py
--- a/pypy/module/pypyjit/test/conftest.py
+++ b/pypy/module/pypyjit/test_pypy_c/conftest.py
@@ -1,7 +1,4 @@
 def pytest_addoption(parser):
     group = parser.getgroup("pypyjit options")
     group.addoption("--pypy", action="store", default=None, dest="pypy_c",
-                    help="DEPRECATED: use this in test_pypy_c instead")
-# XXX kill the corresponding section in the buildbot run,
-# which (as far as I can tell) ignores that option entirely and does
-# the same as the regular py.test.
+                    help="the location of the JIT enabled pypy-c")
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -2,7 +2,7 @@
 import sys, os
 import types
 import subprocess
-import py
+import py, pytest
 from rpython.tool import disassembler
 from rpython.tool.udir import udir
 from rpython.tool import logparser
@@ -15,10 +15,20 @@
     log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
 
     def setup_class(cls):
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("must run this test with pypy")
-        if not sys.pypy_translation_info['translation.jit']:
-            py.test.skip("must give a pypy-c with the jit enabled")
+        pypy_c = pytest.config.option.pypy_c or None
+        if pypy_c is not None:
+            assert os.path.exists(pypy_c), (
+                "--pypy specifies %r, which does not exist" % (pypy_c,))
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print('__pypy__' in sys.builtin_module_names)"])
+            assert 'True' in out, "%r is not a pypy executable" % (pypy_c,)
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print(sys.pypy_translation_info['translation.jit'])"])
+            assert 'True' in out, "%r is a not a JIT-enabled pypy" % (pypy_c,)
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print(sys.version)"])
+            assert out.startswith('3'), "%r is a not a pypy 3" % (pypy_c,)
+        cls.pypy_c = pypy_c
         cls.tmpdir = udir.join('test-pypy-jit')
         cls.tmpdir.ensure(dir=True)
 
@@ -29,6 +39,8 @@
             discard_stdout_before_last_line=False, **jitopts):
         jitopts.setdefault('threshold', 200)
         jitopts.setdefault('disable_unrolling', 9999)
+        if self.pypy_c is None:
+            py.test.skip("run with --pypy=PATH")
         src = py.code.Source(func_or_src)
         if isinstance(func_or_src, types.FunctionType):
             funcname = func_or_src.func_name
@@ -42,12 +54,12 @@
             f.write("import sys\n")
             f.write("sys.setcheckinterval(10000000)\n")
             f.write(str(src) + "\n")
-            f.write("print %s(%s)\n" % (funcname, arglist))
+            f.write("print(%s(%s))\n" % (funcname, arglist))
         #
         # run a child pypy-c with logging enabled
         logfile = self.filepath.new(ext='.log')
         #
-        cmdline = [sys.executable]
+        cmdline = [self.pypy_c]
         if not import_site:
             cmdline.append('-S')
         if jitopts:
@@ -74,6 +86,9 @@
         #if stderr.startswith('debug_alloc.h:'):   # lldebug builds
         #    stderr = ''
         #assert not stderr
+        if not stdout:
+            raise Exception("no stdout produced; stderr='''\n%s'''"
+                            % (stderr,))
         if stderr:
             print '*** stderr of the subprocess: ***'
             print stderr
@@ -433,7 +448,7 @@
         import pytest
         def f():
             import sys
-            print >> sys.stderr, 'SKIP: foobar'
+            sys.stderr.write('SKIP: foobar\n')
         #
         raises(pytest.skip.Exception, "self.run(f, [])")
 

From pypy.commits at gmail.com  Sun Nov 13 08:55:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 05:55:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582870cf.913fc20a.dc058.cec8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88342:b1fc47fa5ded
Date: 2016-11-13 14:05 +0000
http://bitbucket.org/pypy/pypy/changeset/b1fc47fa5ded/

Log:	hg merge default

diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -53,6 +53,19 @@
     def get_opcode_from_info(self, info):
         return self.map[info.bytecode_no]
 
+    @property
+    def filename(self):
+        return self.co.co_filename
+
+    @property
+    def startlineno(self):
+        return self.co.co_firstlineno
+
+    @property
+    def name(self):
+        return self.co.co_name
+
+
 def _setup():
     for opcode in opname:
         if not opcode.startswith('<'):
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -92,15 +92,15 @@
 
 class GenericCode(object):
     def __init__(self, fname, startlineno, name):
-        self._fname = fname
-        self._startlineno = startlineno
-        self._name = name
+        self.filename = fname
+        self.startlineno = startlineno
+        self.name = name
         self._first_bytecodes = {}     # {lineno: info}
         self._source = None
 
     def __repr__(self):
         return 'GenericCode(%r, %r, %r)' % (
-            self._fname, self._startlineno, self._name)
+            self.filename, self.startlineno, self.name)
 
     def get_opcode_from_info(self, info):
         lineno = ~info.bytecode_no
@@ -115,9 +115,9 @@
     @property
     def source(self):
         if self._source is None:
-            src = linecache.getlines(self._fname)
-            if self._startlineno > 0:
-                src = src[self._startlineno - 1:]
+            src = linecache.getlines(self.filename)
+            if self.startlineno > 0:
+                src = src[self.startlineno - 1:]
             self._source = [s.rstrip('\n\r') for s in src]
         return self._source
 

From pypy.commits at gmail.com  Sun Nov 13 08:55:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 05:55:25 -0800 (PST)
Subject: [pypy-commit] pypy default: Updates: give common
 attributes/properties to the two Code classes
Message-ID: <582870cd.c89cc20a.57d54.4916@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88341:e0253fdf011d
Date: 2016-11-13 14:04 +0000
http://bitbucket.org/pypy/pypy/changeset/e0253fdf011d/

Log:	Updates: give common attributes/properties to the two Code classes

diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -53,6 +53,19 @@
     def get_opcode_from_info(self, info):
         return self.map[info.bytecode_no]
 
+    @property
+    def filename(self):
+        return self.co.co_filename
+
+    @property
+    def startlineno(self):
+        return self.co.co_firstlineno
+
+    @property
+    def name(self):
+        return self.co.co_name
+
+
 def _setup():
     for opcode in opname:
         if not opcode.startswith('<'):
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -92,15 +92,15 @@
 
 class GenericCode(object):
     def __init__(self, fname, startlineno, name):
-        self._fname = fname
-        self._startlineno = startlineno
-        self._name = name
+        self.filename = fname
+        self.startlineno = startlineno
+        self.name = name
         self._first_bytecodes = {}     # {lineno: info}
         self._source = None
 
     def __repr__(self):
         return 'GenericCode(%r, %r, %r)' % (
-            self._fname, self._startlineno, self._name)
+            self.filename, self.startlineno, self.name)
 
     def get_opcode_from_info(self, info):
         lineno = ~info.bytecode_no
@@ -115,9 +115,9 @@
     @property
     def source(self):
         if self._source is None:
-            src = linecache.getlines(self._fname)
-            if self._startlineno > 0:
-                src = src[self._startlineno - 1:]
+            src = linecache.getlines(self.filename)
+            if self.startlineno > 0:
+                src = src[self.startlineno - 1:]
             self._source = [s.rstrip('\n\r') for s in src]
         return self._source
 

From pypy.commits at gmail.com  Sun Nov 13 08:55:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 05:55:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: in-progress
Message-ID: <582870d0.e5bfc20a.a6216.cfd0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88343:343bfb83a4bb
Date: 2016-11-13 14:05 +0000
http://bitbucket.org/pypy/pypy/changeset/343bfb83a4bb/

Log:	in-progress

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -11,37 +11,24 @@
 from rpython.tool.jitlogparser.storage import LoopStorage
 
 
-def find_ids_range(code):
+def find_id_linenos(code):
     """
     Parse the given function and return a dictionary mapping "ids" to
-    "line ranges".  Ids are identified by comments with a special syntax::
+    "line number".  Ids are identified by comments with a special syntax::
 
         # "myid" corresponds to the whole line
         print 'foo' # ID: myid
     """
     result = {}
-    start_lineno = code.co.co_firstlineno
+    start_lineno = code.startlineno
     for i, line in enumerate(py.code.Source(code.source)):
         m = re.search('# ID: (\w+)', line)
         if m:
             name = m.group(1)
             lineno = start_lineno+i
-            result[name] = xrange(lineno, lineno+1)
+            result.setdefault(name, lineno)
     return result
 
-def find_ids(code):
-    """
-    Parse the given function and return a dictionary mapping "ids" to
-    "opcodes".
-    """
-    ids = {}
-    ranges = find_ids_range(code)
-    for name, linerange in ranges.iteritems():
-        opcodes = [opcode for opcode in code.opcodes
-                   if opcode.lineno in linerange]
-        ids[name] = opcodes
-    return ids
-
 
 class Log(object):
     def __init__(self, rawtraces):
@@ -107,13 +94,12 @@
     def compute_ids(self, ids):
         #
         # 1. compute the ids of self, i.e. the outer function
-        id2opcodes = find_ids(self.code)
-        all_my_opcodes = self.get_set_of_opcodes()
-        for id, opcodes in id2opcodes.iteritems():
-            if not opcodes:
-                continue
-            target_opcodes = set(opcodes)
-            if all_my_opcodes.intersection(target_opcodes):
+        id2lineno = find_id_linenos(self.code)
+        all_my_opcodes = self.get_list_of_opcodes()
+        for id, lineno in id2lineno.iteritems():
+            opcodes = [opcode for opcode in all_my_opcodes
+                              if opcode.lineno == lineno]
+            if opcodes:
                 ids[id] = opcodes
         #
         # 2. compute the ids of all the inlined functions
@@ -121,12 +107,11 @@
             if isinstance(chunk, TraceWithIds) and chunk.code:
                 chunk.compute_ids(ids)
 
-    def get_set_of_opcodes(self):
-        result = set()
+    def get_list_of_opcodes(self):
+        result = []
         for chunk in self.chunks:
             if isinstance(chunk, TraceForOpcode):
-                opcode = chunk.getopcode()
-                result.add(opcode)
+                result.append(chunk)
         return result
 
     def has_id(self, id):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -3,12 +3,16 @@
 import types
 import subprocess
 import py, pytest
-from rpython.tool import disassembler
 from rpython.tool.udir import udir
 from rpython.tool import logparser
 from rpython.jit.tool.jitoutput import parse_prof
+from rpython.tool.jitlogparser import storage
 from pypy.module.pypyjit.test_pypy_c.model import \
-    Log, find_ids_range, find_ids, OpMatcher, InvalidMatch
+    Log, find_id_linenos, OpMatcher, InvalidMatch
+
+my_file = __file__
+if my_file.endswith('pyc'):
+    my_file = my_file[:-1]
 
 
 class BaseTestPyPyC(object):
@@ -121,31 +125,16 @@
 
 
 class TestLog(object):
-    def test_find_ids_range(self):
+    def test_find_id_linenos(self):
         def f():
             a = 0 # ID: myline
             return a
         #
         start_lineno = f.func_code.co_firstlineno
-        code = disassembler.dis(f)
-        ids = find_ids_range(code)
-        assert len(ids) == 1
-        myline_range = ids['myline']
-        assert list(myline_range) == range(start_lineno+1, start_lineno+2)
-
-    def test_find_ids(self):
-        def f():
-            i = 0
-            x = 0
-            z = x + 3 # ID: myline
-            return z
-        #
-        code = disassembler.dis(f)
-        ids = find_ids(code)
-        assert len(ids) == 1
+        code = storage.GenericCode(my_file, start_lineno, 'f')
+        ids = find_id_linenos(code)
         myline = ids['myline']
-        opcodes_names = [opcode.__class__.__name__ for opcode in myline]
-        assert opcodes_names == ['LOAD_FAST', 'LOAD_CONST', 'BINARY_ADD', 'STORE_FAST']
+        assert myline == start_lineno + 1
 
 
 class TestOpMatcher_(object):
@@ -486,7 +475,7 @@
         log = self.run(f)
         loop, = log.loops_by_id('increment')
         assert loop.filename == self.filepath
-        assert loop.code.co.co_name == 'f'
+        assert loop.code.name == 'f'
         #
         ops = loop.allops()
         assert log.opnames(ops) == [

From pypy.commits at gmail.com  Sun Nov 13 09:47:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:47:22 -0800 (PST)
Subject: [pypy-commit] pypy default: in-progress
Message-ID: <58287cfa.46bb1c0a.31433.4cad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88344:7f3ff26e1ac7
Date: 2016-11-13 14:56 +0000
http://bitbucket.org/pypy/pypy/changeset/7f3ff26e1ac7/

Log:	in-progress

diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -65,6 +65,9 @@
     def name(self):
         return self.co.co_name
 
+    def match_name(self, opcode_name):
+        return self.__class__.__name__ == opcode_name
+
 
 def _setup():
     for opcode in opname:
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -95,7 +95,7 @@
         self.filename = fname
         self.startlineno = startlineno
         self.name = name
-        self._first_bytecodes = {}     # {lineno: info}
+        self._first_bytecodes = {}     # {lineno: bytecode_name}
         self._source = None
 
     def __repr__(self):
@@ -104,13 +104,14 @@
 
     def get_opcode_from_info(self, info):
         lineno = ~info.bytecode_no
-        if self._first_bytecodes.setdefault(lineno, info) is info:
+        bname = info.bytecode_name
+        if self._first_bytecodes.setdefault(lineno, bname) == bname:
             # this is the first opcode of the line---or, at least,
             # the first time we ask for an Opcode on that line.
             line_starts_here = True
         else:
             line_starts_here = False
-        return GenericOpcode(lineno, line_starts_here)
+        return GenericOpcode(lineno, line_starts_here, bname)
 
     @property
     def source(self):
@@ -121,7 +122,32 @@
             self._source = [s.rstrip('\n\r') for s in src]
         return self._source
 
+
 class GenericOpcode(object):
-    def __init__(self, lineno, line_starts_here):
+    def __init__(self, lineno, line_starts_here, bytecode_extra=''):
         self.lineno = lineno
         self.line_starts_here = line_starts_here
+        self.bytecode_extra = bytecode_extra
+
+    def __repr__(self):
+        return 'GenericOpcode(%r, %r, %r)' % (
+            self.lineno, self.line_starts_here, self.bytecode_extra)
+
+    def __eq__(self, other):
+        if not isinstance(other, GenericOpcode):
+            return NotImplemented
+        return (self.lineno == other.lineno and
+                self.line_starts_here == other.line_starts_here and
+                self.bytecode_extra == other.bytecode_extra)
+
+    def __ne__(self, other):
+        if not isinstance(other, GenericOpcode):
+            return NotImplemented
+        return not (self == other)
+
+    def __hash__(self):
+        return hash((self.lineno, self.line_starts_here, self.bytecode_extra))
+
+    def match_name(self, opcode_name):
+        return (self.bytecode_extra == opcode_name or
+                self.bytecode_extra.endswith(' ' + opcode_name))

From pypy.commits at gmail.com  Sun Nov 13 09:47:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:47:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58287cfc.42061c0a.a2bbd.92a7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88345:0f0cf56e8485
Date: 2016-11-13 14:56 +0000
http://bitbucket.org/pypy/pypy/changeset/0f0cf56e8485/

Log:	hg merge default

diff --git a/rpython/tool/disassembler.py b/rpython/tool/disassembler.py
--- a/rpython/tool/disassembler.py
+++ b/rpython/tool/disassembler.py
@@ -65,6 +65,9 @@
     def name(self):
         return self.co.co_name
 
+    def match_name(self, opcode_name):
+        return self.__class__.__name__ == opcode_name
+
 
 def _setup():
     for opcode in opname:
diff --git a/rpython/tool/jitlogparser/storage.py b/rpython/tool/jitlogparser/storage.py
--- a/rpython/tool/jitlogparser/storage.py
+++ b/rpython/tool/jitlogparser/storage.py
@@ -95,7 +95,7 @@
         self.filename = fname
         self.startlineno = startlineno
         self.name = name
-        self._first_bytecodes = {}     # {lineno: info}
+        self._first_bytecodes = {}     # {lineno: bytecode_name}
         self._source = None
 
     def __repr__(self):
@@ -104,13 +104,14 @@
 
     def get_opcode_from_info(self, info):
         lineno = ~info.bytecode_no
-        if self._first_bytecodes.setdefault(lineno, info) is info:
+        bname = info.bytecode_name
+        if self._first_bytecodes.setdefault(lineno, bname) == bname:
             # this is the first opcode of the line---or, at least,
             # the first time we ask for an Opcode on that line.
             line_starts_here = True
         else:
             line_starts_here = False
-        return GenericOpcode(lineno, line_starts_here)
+        return GenericOpcode(lineno, line_starts_here, bname)
 
     @property
     def source(self):
@@ -121,7 +122,32 @@
             self._source = [s.rstrip('\n\r') for s in src]
         return self._source
 
+
 class GenericOpcode(object):
-    def __init__(self, lineno, line_starts_here):
+    def __init__(self, lineno, line_starts_here, bytecode_extra=''):
         self.lineno = lineno
         self.line_starts_here = line_starts_here
+        self.bytecode_extra = bytecode_extra
+
+    def __repr__(self):
+        return 'GenericOpcode(%r, %r, %r)' % (
+            self.lineno, self.line_starts_here, self.bytecode_extra)
+
+    def __eq__(self, other):
+        if not isinstance(other, GenericOpcode):
+            return NotImplemented
+        return (self.lineno == other.lineno and
+                self.line_starts_here == other.line_starts_here and
+                self.bytecode_extra == other.bytecode_extra)
+
+    def __ne__(self, other):
+        if not isinstance(other, GenericOpcode):
+            return NotImplemented
+        return not (self == other)
+
+    def __hash__(self):
+        return hash((self.lineno, self.line_starts_here, self.bytecode_extra))
+
+    def match_name(self, opcode_name):
+        return (self.bytecode_extra == opcode_name or
+                self.bytecode_extra.endswith(' ' + opcode_name))

From pypy.commits at gmail.com  Sun Nov 13 09:47:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:47:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: in-progress
Message-ID: <58287cfe.41a3c20a.927ec.c7da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88346:643d93012403
Date: 2016-11-13 14:57 +0000
http://bitbucket.org/pypy/pypy/changeset/643d93012403/

Log:	in-progress

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -95,10 +95,16 @@
         #
         # 1. compute the ids of self, i.e. the outer function
         id2lineno = find_id_linenos(self.code)
-        all_my_opcodes = self.get_list_of_opcodes()
+        all_my_tracecodes = self.get_list_of_tracecodes()
         for id, lineno in id2lineno.iteritems():
-            opcodes = [opcode for opcode in all_my_opcodes
-                              if opcode.lineno == lineno]
+            seen = set()
+            opcodes = []
+            for tracecode in all_my_tracecodes:
+                if tracecode.lineno == lineno:
+                    opcode = tracecode.getopcode()
+                    if opcode not in seen:
+                        seen.add(opcode)
+                        opcodes.append(opcode)
             if opcodes:
                 ids[id] = opcodes
         #
@@ -107,7 +113,7 @@
             if isinstance(chunk, TraceWithIds) and chunk.code:
                 chunk.compute_ids(ids)
 
-    def get_list_of_opcodes(self):
+    def get_list_of_tracecodes(self):
         result = []
         for chunk in self.chunks:
             if isinstance(chunk, TraceForOpcode):
@@ -129,7 +135,7 @@
         for chunk in self.flatten_chunks():
             opcode = chunk.getopcode()
             if opcode_name is None or \
-                   (opcode and opcode.__class__.__name__ == opcode_name):
+                   (opcode and opcode.match_name(opcode_name)):
                 for op in self._ops_for_chunk(chunk, include_guard_not_invalidated):
                     yield op
             else:
@@ -157,7 +163,7 @@
         for chunk in self.flatten_chunks():
             opcode = chunk.getopcode()
             if opcode in target_opcodes and (opcode_name is None or
-                                             opcode.__class__.__name__ == opcode_name):
+                                             opcode.match_name(opcode_name)):
                 for op in self._ops_for_chunk(chunk, include_guard_not_invalidated):
                     if op in loop_ops:
                         yield op

From pypy.commits at gmail.com  Sun Nov 13 09:55:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:55:47 -0800 (PST)
Subject: [pypy-commit] pypy default: manual copy of dd2ec76131ce: require
 the --pypy option, instead of
Message-ID: <58287ef3.06891c0a.fdec9.ee86@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88347:d9c685ecab5e
Date: 2016-11-13 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/d9c685ecab5e/

Log:	manual copy of dd2ec76131ce: require the --pypy option, instead of
	checking if we're run with "pypy py.test"

diff --git a/pypy/module/pypyjit/test_pypy_c/conftest.py b/pypy/module/pypyjit/test_pypy_c/conftest.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/conftest.py
@@ -0,0 +1,4 @@
+def pytest_addoption(parser):
+    group = parser.getgroup("pypyjit options")
+    group.addoption("--pypy", action="store", default=None, dest="pypy_c",
+                    help="the location of the JIT enabled pypy-c")
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -2,7 +2,7 @@
 import sys, os
 import types
 import subprocess
-import py
+import py, pytest
 from rpython.tool import disassembler
 from rpython.tool.udir import udir
 from rpython.tool import logparser
@@ -15,10 +15,20 @@
     log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
 
     def setup_class(cls):
-        if '__pypy__' not in sys.builtin_module_names:
-            py.test.skip("must run this test with pypy")
-        if not sys.pypy_translation_info['translation.jit']:
-            py.test.skip("must give a pypy-c with the jit enabled")
+        pypy_c = pytest.config.option.pypy_c or None
+        if pypy_c:
+            assert os.path.exists(pypy_c), (
+                "--pypy specifies %r, which does not exist" % (pypy_c,))
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print('__pypy__' in sys.builtin_module_names)"])
+            assert 'True' in out, "%r is not a pypy executable" % (pypy_c,)
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print(sys.pypy_translation_info['translation.jit'])"])
+            assert 'True' in out, "%r is a not a JIT-enabled pypy" % (pypy_c,)
+            out = subprocess.check_output([pypy_c, '-c',
+            "import sys; print(sys.version)"])
+            assert out.startswith('3'), "%r is a not a pypy 3" % (pypy_c,)
+        cls.pypy_c = pypy_c
         cls.tmpdir = udir.join('test-pypy-jit')
         cls.tmpdir.ensure(dir=True)
 
@@ -29,6 +39,8 @@
             discard_stdout_before_last_line=False, **jitopts):
         jitopts.setdefault('threshold', 200)
         jitopts.setdefault('disable_unrolling', 9999)
+        if self.pypy_c is None:
+            py.test.skip("run with --pypy=PATH")
         src = py.code.Source(func_or_src)
         if isinstance(func_or_src, types.FunctionType):
             funcname = func_or_src.func_name
@@ -42,12 +54,12 @@
             f.write("import sys\n")
             f.write("sys.setcheckinterval(10000000)\n")
             f.write(str(src) + "\n")
-            f.write("print %s(%s)\n" % (funcname, arglist))
+            f.write("print(%s(%s))\n" % (funcname, arglist))
         #
         # run a child pypy-c with logging enabled
         logfile = self.filepath.new(ext='.log')
         #
-        cmdline = [sys.executable]
+        cmdline = [self.pypy_c]
         if not import_site:
             cmdline.append('-S')
         if jitopts:
@@ -74,6 +86,9 @@
         #if stderr.startswith('debug_alloc.h:'):   # lldebug builds
         #    stderr = ''
         #assert not stderr
+        if not stdout:
+            raise Exception("no stdout produced; stderr='''\n%s'''"
+                            % (stderr,))
         if stderr:
             print '*** stderr of the subprocess: ***'
             print stderr
@@ -433,7 +448,7 @@
         import pytest
         def f():
             import sys
-            print >> sys.stderr, 'SKIP: foobar'
+            sys.stderr.write('SKIP: foobar\n')
         #
         raises(pytest.skip.Exception, "self.run(f, [])")
 

From pypy.commits at gmail.com  Sun Nov 13 09:56:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:50 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Create union() as a
 side-effect-free binary function to replace unionof() wherever possible
Message-ID: <58287f32.c1341c0a.942da.20d9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88348:6639e3b4a1aa
Date: 2016-09-02 19:35 +0100
http://bitbucket.org/pypy/pypy/changeset/6639e3b4a1aa/

Log:	Create union() as a side-effect-free binary function to replace
	unionof() wherever possible

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -94,16 +94,9 @@
         if self == other:
             return True
         try:
-            TLS.no_side_effects_in_union += 1
-        except AttributeError:
-            TLS.no_side_effects_in_union = 1
-        try:
-            try:
-                return pair(self, other).union() == self
-            except UnionError:
-                return False
-        finally:
-            TLS.no_side_effects_in_union -= 1
+            return union(self, other) == self
+        except UnionError:
+            return False
 
     def is_constant(self):
         d = self.__dict__
@@ -739,6 +732,23 @@
     def __repr__(self):
         return str(self)
 
+def union(s1, s2):
+    """The join operation in the lattice of annotations.
+
+    It is the most precise SomeObject instance that contains both arguments.
+
+    union() is (supposed to be) idempotent, commutative, associative and has
+    no side-effects.
+    """
+    try:
+        TLS.no_side_effects_in_union += 1
+    except AttributeError:
+        TLS.no_side_effects_in_union = 1
+    try:
+        return pair(s1, s2).union()
+    finally:
+        TLS.no_side_effects_in_union -= 1
+
 def unionof(*somevalues):
     "The most precise SomeValue instance that contains all the values."
     try:

From pypy.commits at gmail.com  Sun Nov 13 09:56:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:52 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Use union() instead of
 unionof() in a few places
Message-ID: <58287f34.624fc20a.f7b3f.a6ff@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88349:82a591a821b7
Date: 2016-09-02 21:40 +0100
http://bitbucket.org/pypy/pypy/changeset/82a591a821b7/

Log:	Use union() instead of unionof() in a few places

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -246,7 +246,7 @@
         if s_old is not None:
             if not s_value.contains(s_old):
                 log.WARNING("%s does not contain %s" % (s_value, s_old))
-                log.WARNING("%s" % annmodel.unionof(s_value, s_old))
+                log.WARNING("%s" % annmodel.union(s_value, s_old))
                 assert False
         arg.annotation = s_value
 
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -11,7 +11,7 @@
     SomeBuiltinMethod, SomeIterator, SomePBC, SomeNone, SomeFloat, s_None,
     SomeByteArray, SomeWeakRef, SomeSingleFloat,
     SomeLongFloat, SomeType, SomeTypeOf, SomeConstantType, unionof, UnionError,
-    read_can_only_throw, add_knowntypedata,
+    union, read_can_only_throw, add_knowntypedata,
     merge_knowntypedata,)
 from rpython.annotator.bookkeeper import immutablevalue, getbookkeeper
 from rpython.flowspace.model import Variable, Constant, const
@@ -703,13 +703,13 @@
         pairtype(SomeException, SomeInstance),
         pairtype(SomeException, SomeNone)):
     def union((s_exc, s_inst)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(
         pairtype(SomeInstance, SomeException),
         pairtype(SomeNone, SomeException)):
     def union((s_inst, s_exc)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(pairtype(SomeException, SomeException)):
     def union((s_exc1, s_exc2)):
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -6,7 +6,7 @@
 
 from rpython.annotator.model import (
     SomeInteger, SomeChar, SomeBool, SomeString, SomeTuple,
-    SomeUnicodeCodePoint, SomeFloat, unionof, SomeUnicodeString,
+    SomeUnicodeCodePoint, SomeFloat, union, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
     SomeOrderedDict, SomeByteArray, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import (
@@ -166,14 +166,14 @@
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        return unionof(*s_values)
+        return union(*s_values)
 
 def builtin_max(*s_values):
     if len(s_values) == 1: # xxx do we support this?
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        s = unionof(*s_values)
+        s = union(*s_values)
         if type(s) is SomeInteger and not s.nonneg:
             nonneg = False
             for s1 in s_values:
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -1,5 +1,5 @@
-from rpython.annotator.model import s_ImpossibleValue
-from rpython.annotator.model import SomeInteger, s_Bool, unionof
+from rpython.annotator.model import (
+    s_ImpossibleValue, SomeInteger, s_Bool, union)
 from rpython.annotator.listdef import ListItem
 from rpython.rlib.objectmodel import compute_hash
 
@@ -34,8 +34,8 @@
 
     def update_rdict_annotations(self, s_eqfn, s_hashfn, other=None):
         assert self.custom_eq_hash
-        s_eqfn = unionof(s_eqfn, self.s_rdict_eqfn)
-        s_hashfn = unionof(s_hashfn, self.s_rdict_hashfn)
+        s_eqfn = union(s_eqfn, self.s_rdict_eqfn)
+        s_hashfn = union(s_hashfn, self.s_rdict_hashfn)
         self.s_rdict_eqfn = s_eqfn
         self.s_rdict_hashfn = s_hashfn
         self.emulate_rdict_calls(other=other)
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -146,14 +146,14 @@
     someinst = lambda cls, **kw: SomeInstance(bk.getuniqueclassdef(cls), **kw)
     s_inst = someinst(Exception)
     s_exc = bk.new_exception([ValueError, IndexError])
-    assert unionof(s_exc, s_inst) == s_inst
-    assert unionof(s_inst, s_exc) == s_inst
-    s_nullable = unionof(s_None, bk.new_exception([ValueError]))
+    assert union(s_exc, s_inst) == s_inst
+    assert union(s_inst, s_exc) == s_inst
+    s_nullable = union(s_None, bk.new_exception([ValueError]))
     assert isinstance(s_nullable, SomeInstance)
     assert s_nullable.can_be_None
     s_exc1 = bk.new_exception([ValueError])
     s_exc2 = bk.new_exception([IndexError])
-    unionof(s_exc1, s_exc2) == unionof(s_exc2, s_exc1)
+    union(s_exc1, s_exc2) == union(s_exc2, s_exc1)
 
 def contains_s(s_a, s_b):
     if s_b is None:
diff --git a/rpython/rtyper/test/test_llannotation.py b/rpython/rtyper/test/test_llannotation.py
--- a/rpython/rtyper/test/test_llannotation.py
+++ b/rpython/rtyper/test/test_llannotation.py
@@ -1,6 +1,6 @@
 import py.test
 from rpython.annotator.model import (
-    SomeInteger, SomeBool, SomeChar, unionof, SomeImpossibleValue,
+    SomeInteger, SomeBool, SomeChar, union, SomeImpossibleValue,
     UnionError, SomeInstance, SomeSingleFloat)
 from rpython.rlib.rarithmetic import r_uint, r_singlefloat
 from rpython.rtyper.llannotation import (
@@ -69,22 +69,22 @@
     PA1 = lltype.Ptr(lltype.GcArray())
     PA2 = lltype.Ptr(lltype.GcArray())
 
-    assert unionof(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
 
-    assert unionof(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
 
-    assert unionof(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
-    assert unionof(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
+    assert union(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
 
     with py.test.raises(UnionError):
-        unionof(SomePtr(PA1), SomePtr(PS1))
+        union(SomePtr(PA1), SomePtr(PS1))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomePtr(PS3))
+        union(SomePtr(PS1), SomePtr(PS3))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomeInteger())
+        union(SomePtr(PS1), SomeInteger())
     with py.test.raises(UnionError):
-        unionof(SomeInteger(), SomePtr(PS1))
+        union(SomeInteger(), SomePtr(PS1))

From pypy.commits at gmail.com  Sun Nov 13 09:56:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:54 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Add an elementary (but
 failing) hypothesis test for union()
Message-ID: <58287f36.542e1c0a.32a09.b6ec@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88350:fad883de71cf
Date: 2016-09-03 01:24 +0100
http://bitbucket.org/pypy/pypy/changeset/fad883de71cf/

Log:	Add an elementary (but failing) hypothesis test for union()

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -1,5 +1,8 @@
 import pytest
 
+from hypothesis import given
+from hypothesis import strategies as st
+
 from rpython.flowspace.model import Variable
 from rpython.flowspace.operation import op
 from rpython.translator.translator import TranslationContext
@@ -102,6 +105,21 @@
     assert f2.contains(f1)
     assert f1.contains(f2)
 
+def const_int(n):
+    s = SomeInteger(nonneg=(n>=0))
+    s.const = n
+    return s
+
+st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
+                   st.builds(const_int, st.integers()))
+st_annotation = st_int
+
+ at given(s=st_annotation)
+def test_union_unary(s):
+    assert union(s, s) == s
+    assert union(s_ImpossibleValue, s) == s
+
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)

From pypy.commits at gmail.com  Sun Nov 13 09:56:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:55 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Fix union() and give a
 correct explanation for the s1==s2 case
Message-ID: <58287f37.c6bdc20a.e3d52.ded8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88351:bd63a3f6c556
Date: 2016-09-03 01:44 +0100
http://bitbucket.org/pypy/pypy/changeset/bd63a3f6c556/

Log:	Fix union() and give a correct explanation for the s1==s2 case

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -745,6 +745,10 @@
     except AttributeError:
         TLS.no_side_effects_in_union = 1
     try:
+        if s1 == s2:
+            # Most pair(...).union() methods deal incorrectly with that case
+            # when constants are involved.
+            return s1
         return pair(s1, s2).union()
     finally:
         TLS.no_side_effects_in_union -= 1
@@ -759,8 +763,7 @@
             if s1 != s2:
                 s1 = pair(s1, s2).union()
     else:
-        # this is just a performance shortcut
-        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
+        # See comment in union() above
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1

From pypy.commits at gmail.com  Sun Nov 13 09:56:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:57 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Expand hypothesis testing
 of union() until it fails
Message-ID: <58287f39.45f6c20a.47cb3.19e7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88352:131c9aecb309
Date: 2016-09-03 03:41 +0100
http://bitbucket.org/pypy/pypy/changeset/131c9aecb309/

Log:	Expand hypothesis testing of union() until it fails

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -673,6 +673,10 @@
 
 s_None = SomeNone()
 s_Bool = SomeBool()
+s_True = SomeBool()
+s_True.const = True
+s_False = SomeBool()
+s_False.const = False
 s_Int = SomeInteger()
 s_ImpossibleValue = SomeImpossibleValue()
 s_Str0 = SomeString(no_nul=True)
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -70,11 +70,11 @@
     except TypeError:    # if A0 is also a new-style class, e.g. in PyPy
         class B3(A0, object):
             pass
-    assert commonbase(A1,A2) is A0
-    assert commonbase(A1,A0) is A0
-    assert commonbase(A1,A1) is A1
-    assert commonbase(A2,B2) is object
-    assert commonbase(A2,B3) is A0
+    assert commonbase(A1, A2) is A0
+    assert commonbase(A1, A0) is A0
+    assert commonbase(A1, A1) is A1
+    assert commonbase(A2, B2) is object
+    assert commonbase(A2, B3) is A0
 
 def test_list_union():
     listdef1 = ListDef('dummy', SomeInteger(nonneg=True))
@@ -105,20 +105,89 @@
     assert f2.contains(f1)
     assert f1.contains(f2)
 
+def const_float(x):
+    s = SomeFloat()
+    s.const = x
+    return s
+
 def const_int(n):
-    s = SomeInteger(nonneg=(n>=0))
+    s = SomeInteger(nonneg=(n >= 0))
     s.const = n
     return s
 
+def const_str(x):
+    no_nul = not '\x00' in x
+    if len(x) == 1:
+        result = SomeChar(no_nul=no_nul)
+    else:
+        result = SomeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def const_unicode(x):
+    no_nul = not u'\x00' in x
+    if len(x) == 1:
+        result = SomeUnicodeCodePoint(no_nul=no_nul)
+    else:
+        result = SomeUnicodeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def compatible(s1, s2):
+    try:
+        union(s1, s2)
+    except UnionError:
+        return False
+    return True
+
+def compatible_pair(pair_s):
+    return compatible(*pair_s)
+
+st_float = st.just(SomeFloat()) | st.builds(const_float, st.floats())
 st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
                    st.builds(const_int, st.integers()))
-st_annotation = st_int
+st_bool = st.sampled_from([s_Bool, s_True, s_False])
+st_numeric = st.one_of(st_float, st_int, st_bool)
+st_str = (st.builds(SomeString, st.booleans(), st.booleans())
+          | st.builds(const_str, st.binary()))
+st_unicode = (st.builds(SomeUnicodeString, st.booleans(), st.booleans())
+              | st.builds(const_unicode, st.text()))
+st_simple = st.one_of(st_numeric, st_str, st_unicode, st.just(s_ImpossibleValue), st.just(s_None))
 
- at given(s=st_annotation)
+def valid_unions(st_ann):
+    """From a strategy generating annotations, create a strategy returning
+    unions of these annotations."""
+    pairs = st.tuples(st_ann, st_ann)
+    return pairs.filter(compatible_pair).map(lambda t: union(*t))
+
+
+st_annotation = st.recursive(st_simple,
+    lambda st_ann: valid_unions(st_ann) | st.builds(SomeTuple, st.lists(st_ann)),
+    max_leaves=3)
+
+ at given(s=st_numeric)
 def test_union_unary(s):
     assert union(s, s) == s
     assert union(s_ImpossibleValue, s) == s
 
+ at given(s1=st_numeric, s2=st_numeric)
+def test_commutativity_of_union_compatibility(s1, s2):
+    assert compatible(s1, s2) == compatible(s2, s1)
+
+ at given(st.tuples(st_annotation, st_annotation).filter(lambda t: compatible(*t)))
+def test_union_commutative(t):
+    s1, s2 = t
+    s_union = union(s1, s2)
+    assert union(s2, s1) == s_union
+    assert s_union.contains(s1)
+    assert s_union.contains(s2)
+
+ at given(st.tuples(st_annotation, st_annotation, st_annotation).filter(
+    lambda t: compatible(t[0], t[1]) and compatible(t[1], t[2]) and compatible(t[0], t[2])))
+def test_union_associative(t):
+    s1, s2, s3 = t
+    assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
+
 
 def compile_function(function, annotation=[]):
     t = TranslationContext()

From pypy.commits at gmail.com  Sun Nov 13 09:56:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 06:56:59 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Improve and simplify
 hypothesis tests
Message-ID: <58287f3b.c4811c0a.87ceb.1cce@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88353:76d338a9c8b1
Date: 2016-09-06 15:38 +0100
http://bitbucket.org/pypy/pypy/changeset/76d338a9c8b1/

Log:	Improve and simplify hypothesis tests

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -1,6 +1,6 @@
 import pytest
 
-from hypothesis import given
+from hypothesis import given, assume, settings
 from hypothesis import strategies as st
 
 from rpython.flowspace.model import Variable
@@ -165,27 +165,30 @@
     lambda st_ann: valid_unions(st_ann) | st.builds(SomeTuple, st.lists(st_ann)),
     max_leaves=3)
 
- at given(s=st_numeric)
+ at given(s=st_annotation)
 def test_union_unary(s):
     assert union(s, s) == s
     assert union(s_ImpossibleValue, s) == s
 
- at given(s1=st_numeric, s2=st_numeric)
+ at given(s1=st_annotation, s2=st_annotation)
 def test_commutativity_of_union_compatibility(s1, s2):
     assert compatible(s1, s2) == compatible(s2, s1)
 
- at given(st.tuples(st_annotation, st_annotation).filter(lambda t: compatible(*t)))
-def test_union_commutative(t):
-    s1, s2 = t
-    s_union = union(s1, s2)
+ at given(st_annotation, st_annotation)
+def test_union_commutative(s1, s2):
+    try:
+        s_union = union(s1, s2)
+    except UnionError:
+        assume(False)
     assert union(s2, s1) == s_union
     assert s_union.contains(s1)
     assert s_union.contains(s2)
 
- at given(st.tuples(st_annotation, st_annotation, st_annotation).filter(
-    lambda t: compatible(t[0], t[1]) and compatible(t[1], t[2]) and compatible(t[0], t[2])))
-def test_union_associative(t):
-    s1, s2, s3 = t
+ at pytest.mark.xfail
+ at settings(max_examples=500)
+ at given(st_annotation, st_annotation, st_annotation)
+def test_union_associative(s1, s2, s3):
+    assume(compatible(s1, s2) and compatible(union(s1, s2), s3))
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 

From pypy.commits at gmail.com  Sun Nov 13 09:57:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:57:02 -0800 (PST)
Subject: [pypy-commit] buildbot default: update:
Message-ID: <58287f3e.0777c20a.ec165.6c7a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1018:27fa8b5fac2b
Date: 2016-11-13 15:56 +0100
http://bitbucket.org/pypy/buildbot/changeset/27fa8b5fac2b/

Log:	update:

	* pypy/module/test/: don't run it again with the '--pypy' option,
	which has been pointless since a while

	* pypy/module/test_pypy_c/: run it with the '--pypy' option instead
	of using 'pypy py.test'. This will only work on modern pypy's with
	the changeset d9c685ecab5e or dd2ec76131ce.

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -466,15 +466,15 @@
         # kill this step when the transition to test_pypy_c_new has been
         # completed
         # "old" test_pypy_c
-        factory.addStep(PytestCmd(
-            description="pypyjit tests",
-            command=prefix + ["python", "pypy/test_all.py",
-                     "--pypy=pypy/goal/pypy-c",
-                     "--resultlog=pypyjit.log",
-                     "pypy/module/pypyjit/test"],
-            logfiles={'pytestLog': 'pypyjit.log'},
-            env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
-                }))
+        #factory.addStep(PytestCmd(
+        #    description="pypyjit tests",
+        #    command=prefix + ["python", "pypy/test_all.py",
+        #             "--pypy=pypy/goal/pypy-c",
+        #             "--resultlog=pypyjit.log",
+        #             "pypy/module/pypyjit/test"],
+        #    logfiles={'pytestLog': 'pypyjit.log'},
+        #    env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
+        #        }))
         #
         # "new" test_pypy_c
         if platform == 'win32':
@@ -483,7 +483,8 @@
             cmd = 'pypy/goal/pypy-c'
         factory.addStep(PytestCmd(
             description="pypyjit tests",
-            command=prefix + [cmd, "pypy/test_all.py",
+            command=prefix + ["python", "pypy/test_all.py",
+                     "--pypy=%s" % (cmd,),
                      "--resultlog=pypyjit_new.log",
                      "pypy/module/pypyjit/test_pypy_c"],
             logfiles={'pytestLog': 'pypyjit_new.log'},

From pypy.commits at gmail.com  Sun Nov 13 09:59:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 06:59:39 -0800 (PST)
Subject: [pypy-commit] pypy default: backward compatibility: if we don't
 give --pypy, use sys.executable if that's a pypy with jit
Message-ID: <58287fdb.44e61c0a.dbd1e.3629@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88354:02aa2a6c1bbf
Date: 2016-11-13 15:59 +0100
http://bitbucket.org/pypy/pypy/changeset/02aa2a6c1bbf/

Log:	backward compatibility: if we don't give --pypy, use sys.executable
	if that's a pypy with jit

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -28,6 +28,12 @@
             out = subprocess.check_output([pypy_c, '-c',
             "import sys; print(sys.version)"])
             assert out.startswith('3'), "%r is a not a pypy 3" % (pypy_c,)
+        else:
+            # backward compatibility: use pypy_c = sys.executable
+            # if that's a pypy with a JIT
+            if ('__pypy__' in sys.builtin_module_names and
+                sys.pypy_translation_info['translation.jit']):
+                pypy_c = sys.executable
         cls.pypy_c = pypy_c
         cls.tmpdir = udir.join('test-pypy-jit')
         cls.tmpdir.ensure(dir=True)

From pypy.commits at gmail.com  Sun Nov 13 10:05:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 07:05:07 -0800 (PST)
Subject: [pypy-commit] pypy default: oups
Message-ID: <58288123.471ec20a.2f6d8.dd72@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88355:e3e05182dc35
Date: 2016-11-13 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/e3e05182dc35/

Log:	oups

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -27,7 +27,7 @@
             assert 'True' in out, "%r is a not a JIT-enabled pypy" % (pypy_c,)
             out = subprocess.check_output([pypy_c, '-c',
             "import sys; print(sys.version)"])
-            assert out.startswith('3'), "%r is a not a pypy 3" % (pypy_c,)
+            assert out.startswith('2.7'), "%r is a not a pypy 2.7" % (pypy_c,)
         else:
             # backward compatibility: use pypy_c = sys.executable
             # if that's a pypy with a JIT

From pypy.commits at gmail.com  Sun Nov 13 10:06:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 07:06:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5828817f.a285c20a.6e708.d924@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88356:bcac3120dee2
Date: 2016-11-13 16:05 +0100
http://bitbucket.org/pypy/pypy/changeset/bcac3120dee2/

Log:	hg merge default


From pypy.commits at gmail.com  Sun Nov 13 10:27:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 07:27:24 -0800 (PST)
Subject: [pypy-commit] pypy default: Revert d9c685ecab5e: doesn't work. On
 pypy2 we have to run "pypy py.test".
Message-ID: <5828865c.c5371c0a.818b2.8485@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88357:14917346434c
Date: 2016-11-13 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/14917346434c/

Log:	Revert d9c685ecab5e: doesn't work. On pypy2 we have to run "pypy
	py.test". On pypy3 we cannot run "pypy py.test". Fun.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -2,7 +2,7 @@
 import sys, os
 import types
 import subprocess
-import py, pytest
+import py
 from rpython.tool import disassembler
 from rpython.tool.udir import udir
 from rpython.tool import logparser
@@ -15,26 +15,10 @@
     log_string = 'jit-log-opt,jit-log-noopt,jit-log-virtualstate,jit-summary'
 
     def setup_class(cls):
-        pypy_c = pytest.config.option.pypy_c or None
-        if pypy_c:
-            assert os.path.exists(pypy_c), (
-                "--pypy specifies %r, which does not exist" % (pypy_c,))
-            out = subprocess.check_output([pypy_c, '-c',
-            "import sys; print('__pypy__' in sys.builtin_module_names)"])
-            assert 'True' in out, "%r is not a pypy executable" % (pypy_c,)
-            out = subprocess.check_output([pypy_c, '-c',
-            "import sys; print(sys.pypy_translation_info['translation.jit'])"])
-            assert 'True' in out, "%r is a not a JIT-enabled pypy" % (pypy_c,)
-            out = subprocess.check_output([pypy_c, '-c',
-            "import sys; print(sys.version)"])
-            assert out.startswith('2.7'), "%r is a not a pypy 2.7" % (pypy_c,)
-        else:
-            # backward compatibility: use pypy_c = sys.executable
-            # if that's a pypy with a JIT
-            if ('__pypy__' in sys.builtin_module_names and
-                sys.pypy_translation_info['translation.jit']):
-                pypy_c = sys.executable
-        cls.pypy_c = pypy_c
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("must run this test with pypy")
+        if not sys.pypy_translation_info['translation.jit']:
+            py.test.skip("must give a pypy-c with the jit enabled")
         cls.tmpdir = udir.join('test-pypy-jit')
         cls.tmpdir.ensure(dir=True)
 
@@ -45,8 +29,6 @@
             discard_stdout_before_last_line=False, **jitopts):
         jitopts.setdefault('threshold', 200)
         jitopts.setdefault('disable_unrolling', 9999)
-        if self.pypy_c is None:
-            py.test.skip("run with --pypy=PATH")
         src = py.code.Source(func_or_src)
         if isinstance(func_or_src, types.FunctionType):
             funcname = func_or_src.func_name
@@ -60,12 +42,12 @@
             f.write("import sys\n")
             f.write("sys.setcheckinterval(10000000)\n")
             f.write(str(src) + "\n")
-            f.write("print(%s(%s))\n" % (funcname, arglist))
+            f.write("print %s(%s)\n" % (funcname, arglist))
         #
         # run a child pypy-c with logging enabled
         logfile = self.filepath.new(ext='.log')
         #
-        cmdline = [self.pypy_c]
+        cmdline = [sys.executable]
         if not import_site:
             cmdline.append('-S')
         if jitopts:
@@ -92,9 +74,6 @@
         #if stderr.startswith('debug_alloc.h:'):   # lldebug builds
         #    stderr = ''
         #assert not stderr
-        if not stdout:
-            raise Exception("no stdout produced; stderr='''\n%s'''"
-                            % (stderr,))
         if stderr:
             print '*** stderr of the subprocess: ***'
             print stderr
@@ -454,7 +433,7 @@
         import pytest
         def f():
             import sys
-            sys.stderr.write('SKIP: foobar\n')
+            print >> sys.stderr, 'SKIP: foobar'
         #
         raises(pytest.skip.Exception, "self.run(f, [])")
 

From pypy.commits at gmail.com  Sun Nov 13 10:27:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 07:27:59 -0800 (PST)
Subject: [pypy-commit] pypy default: remove this file too
Message-ID: <5828867f.0370c20a.98ff1.e37a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88358:0e136ff64a98
Date: 2016-11-13 16:27 +0100
http://bitbucket.org/pypy/pypy/changeset/0e136ff64a98/

Log:	remove this file too

diff --git a/pypy/module/pypyjit/test_pypy_c/conftest.py b/pypy/module/pypyjit/test_pypy_c/conftest.py
deleted file mode 100644
--- a/pypy/module/pypyjit/test_pypy_c/conftest.py
+++ /dev/null
@@ -1,4 +0,0 @@
-def pytest_addoption(parser):
-    group = parser.getgroup("pypyjit options")
-    group.addoption("--pypy", action="store", default=None, dest="pypy_c",
-                    help="the location of the JIT enabled pypy-c")

From pypy.commits at gmail.com  Sun Nov 13 10:35:46 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 07:35:46 -0800 (PST)
Subject: [pypy-commit] pypy union-side-effects-2: Close branch
 union-side-effects-2
Message-ID: <58288852.233ac20a.6de12.e77a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: union-side-effects-2
Changeset: r88359:8251bfceb019
Date: 2016-11-13 15:35 +0000
http://bitbucket.org/pypy/pypy/changeset/8251bfceb019/

Log:	Close branch union-side-effects-2


From pypy.commits at gmail.com  Sun Nov 13 10:36:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 07:36:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in union-side-effects-2 (pull
 request #495)
Message-ID: <58288861.c6bdc20a.e3d52.ea09@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88360:c14a4a09d0fe
Date: 2016-11-13 15:35 +0000
http://bitbucket.org/pypy/pypy/changeset/c14a4a09d0fe/

Log:	Merged in union-side-effects-2 (pull request #495)

	Creates and tests union().

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -246,7 +246,7 @@
         if s_old is not None:
             if not s_value.contains(s_old):
                 log.WARNING("%s does not contain %s" % (s_value, s_old))
-                log.WARNING("%s" % annmodel.unionof(s_value, s_old))
+                log.WARNING("%s" % annmodel.union(s_value, s_old))
                 assert False
         arg.annotation = s_value
 
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -11,7 +11,7 @@
     SomeBuiltinMethod, SomeIterator, SomePBC, SomeNone, SomeFloat, s_None,
     SomeByteArray, SomeWeakRef, SomeSingleFloat,
     SomeLongFloat, SomeType, SomeTypeOf, SomeConstantType, unionof, UnionError,
-    read_can_only_throw, add_knowntypedata,
+    union, read_can_only_throw, add_knowntypedata,
     merge_knowntypedata,)
 from rpython.annotator.bookkeeper import immutablevalue, getbookkeeper
 from rpython.flowspace.model import Variable, Constant, const
@@ -703,13 +703,13 @@
         pairtype(SomeException, SomeInstance),
         pairtype(SomeException, SomeNone)):
     def union((s_exc, s_inst)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(
         pairtype(SomeInstance, SomeException),
         pairtype(SomeNone, SomeException)):
     def union((s_inst, s_exc)):
-        return unionof(s_exc.as_SomeInstance(), s_inst)
+        return union(s_exc.as_SomeInstance(), s_inst)
 
 class __extend__(pairtype(SomeException, SomeException)):
     def union((s_exc1, s_exc2)):
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -6,7 +6,7 @@
 
 from rpython.annotator.model import (
     SomeInteger, SomeChar, SomeBool, SomeString, SomeTuple,
-    SomeUnicodeCodePoint, SomeFloat, unionof, SomeUnicodeString,
+    SomeUnicodeCodePoint, SomeFloat, union, SomeUnicodeString,
     SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator,
     SomeOrderedDict, SomeByteArray, add_knowntypedata, s_ImpossibleValue,)
 from rpython.annotator.bookkeeper import (
@@ -166,14 +166,14 @@
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        return unionof(*s_values)
+        return union(*s_values)
 
 def builtin_max(*s_values):
     if len(s_values) == 1: # xxx do we support this?
         s_iter = s_values[0].iter()
         return s_iter.next()
     else:
-        s = unionof(*s_values)
+        s = union(*s_values)
         if type(s) is SomeInteger and not s.nonneg:
             nonneg = False
             for s1 in s_values:
diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -1,5 +1,5 @@
-from rpython.annotator.model import s_ImpossibleValue
-from rpython.annotator.model import SomeInteger, s_Bool, unionof
+from rpython.annotator.model import (
+    s_ImpossibleValue, SomeInteger, s_Bool, union)
 from rpython.annotator.listdef import ListItem
 from rpython.rlib.objectmodel import compute_hash
 
@@ -34,8 +34,8 @@
 
     def update_rdict_annotations(self, s_eqfn, s_hashfn, other=None):
         assert self.custom_eq_hash
-        s_eqfn = unionof(s_eqfn, self.s_rdict_eqfn)
-        s_hashfn = unionof(s_hashfn, self.s_rdict_hashfn)
+        s_eqfn = union(s_eqfn, self.s_rdict_eqfn)
+        s_hashfn = union(s_hashfn, self.s_rdict_hashfn)
         self.s_rdict_eqfn = s_eqfn
         self.s_rdict_hashfn = s_hashfn
         self.emulate_rdict_calls(other=other)
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -94,16 +94,9 @@
         if self == other:
             return True
         try:
-            TLS.no_side_effects_in_union += 1
-        except AttributeError:
-            TLS.no_side_effects_in_union = 1
-        try:
-            try:
-                return pair(self, other).union() == self
-            except UnionError:
-                return False
-        finally:
-            TLS.no_side_effects_in_union -= 1
+            return union(self, other) == self
+        except UnionError:
+            return False
 
     def is_constant(self):
         d = self.__dict__
@@ -680,6 +673,10 @@
 
 s_None = SomeNone()
 s_Bool = SomeBool()
+s_True = SomeBool()
+s_True.const = True
+s_False = SomeBool()
+s_False.const = False
 s_Int = SomeInteger()
 s_ImpossibleValue = SomeImpossibleValue()
 s_Str0 = SomeString(no_nul=True)
@@ -739,6 +736,27 @@
     def __repr__(self):
         return str(self)
 
+def union(s1, s2):
+    """The join operation in the lattice of annotations.
+
+    It is the most precise SomeObject instance that contains both arguments.
+
+    union() is (supposed to be) idempotent, commutative, associative and has
+    no side-effects.
+    """
+    try:
+        TLS.no_side_effects_in_union += 1
+    except AttributeError:
+        TLS.no_side_effects_in_union = 1
+    try:
+        if s1 == s2:
+            # Most pair(...).union() methods deal incorrectly with that case
+            # when constants are involved.
+            return s1
+        return pair(s1, s2).union()
+    finally:
+        TLS.no_side_effects_in_union -= 1
+
 def unionof(*somevalues):
     "The most precise SomeValue instance that contains all the values."
     try:
@@ -749,8 +767,7 @@
             if s1 != s2:
                 s1 = pair(s1, s2).union()
     else:
-        # this is just a performance shortcut
-        # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
+        # See comment in union() above
         if s1 != s2:
             s1 = pair(s1, s2).union()
     return s1
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -1,5 +1,8 @@
 import pytest
 
+from hypothesis import given, assume, settings
+from hypothesis import strategies as st
+
 from rpython.flowspace.model import Variable
 from rpython.flowspace.operation import op
 from rpython.translator.translator import TranslationContext
@@ -67,11 +70,11 @@
     except TypeError:    # if A0 is also a new-style class, e.g. in PyPy
         class B3(A0, object):
             pass
-    assert commonbase(A1,A2) is A0
-    assert commonbase(A1,A0) is A0
-    assert commonbase(A1,A1) is A1
-    assert commonbase(A2,B2) is object
-    assert commonbase(A2,B3) is A0
+    assert commonbase(A1, A2) is A0
+    assert commonbase(A1, A0) is A0
+    assert commonbase(A1, A1) is A1
+    assert commonbase(A2, B2) is object
+    assert commonbase(A2, B3) is A0
 
 def test_list_union():
     listdef1 = ListDef('dummy', SomeInteger(nonneg=True))
@@ -102,6 +105,93 @@
     assert f2.contains(f1)
     assert f1.contains(f2)
 
+def const_float(x):
+    s = SomeFloat()
+    s.const = x
+    return s
+
+def const_int(n):
+    s = SomeInteger(nonneg=(n >= 0))
+    s.const = n
+    return s
+
+def const_str(x):
+    no_nul = not '\x00' in x
+    if len(x) == 1:
+        result = SomeChar(no_nul=no_nul)
+    else:
+        result = SomeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def const_unicode(x):
+    no_nul = not u'\x00' in x
+    if len(x) == 1:
+        result = SomeUnicodeCodePoint(no_nul=no_nul)
+    else:
+        result = SomeUnicodeString(no_nul=no_nul)
+    result.const = x
+    return result
+
+def compatible(s1, s2):
+    try:
+        union(s1, s2)
+    except UnionError:
+        return False
+    return True
+
+def compatible_pair(pair_s):
+    return compatible(*pair_s)
+
+st_float = st.just(SomeFloat()) | st.builds(const_float, st.floats())
+st_int = st.one_of(st.builds(SomeInteger, st.booleans(), st.booleans()),
+                   st.builds(const_int, st.integers()))
+st_bool = st.sampled_from([s_Bool, s_True, s_False])
+st_numeric = st.one_of(st_float, st_int, st_bool)
+st_str = (st.builds(SomeString, st.booleans(), st.booleans())
+          | st.builds(const_str, st.binary()))
+st_unicode = (st.builds(SomeUnicodeString, st.booleans(), st.booleans())
+              | st.builds(const_unicode, st.text()))
+st_simple = st.one_of(st_numeric, st_str, st_unicode, st.just(s_ImpossibleValue), st.just(s_None))
+
+def valid_unions(st_ann):
+    """From a strategy generating annotations, create a strategy returning
+    unions of these annotations."""
+    pairs = st.tuples(st_ann, st_ann)
+    return pairs.filter(compatible_pair).map(lambda t: union(*t))
+
+
+st_annotation = st.recursive(st_simple,
+    lambda st_ann: valid_unions(st_ann) | st.builds(SomeTuple, st.lists(st_ann)),
+    max_leaves=3)
+
+ at given(s=st_annotation)
+def test_union_unary(s):
+    assert union(s, s) == s
+    assert union(s_ImpossibleValue, s) == s
+
+ at given(s1=st_annotation, s2=st_annotation)
+def test_commutativity_of_union_compatibility(s1, s2):
+    assert compatible(s1, s2) == compatible(s2, s1)
+
+ at given(st_annotation, st_annotation)
+def test_union_commutative(s1, s2):
+    try:
+        s_union = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assert union(s2, s1) == s_union
+    assert s_union.contains(s1)
+    assert s_union.contains(s2)
+
+ at pytest.mark.xfail
+ at settings(max_examples=500)
+ at given(st_annotation, st_annotation, st_annotation)
+def test_union_associative(s1, s2, s3):
+    assume(compatible(s1, s2) and compatible(union(s1, s2), s3))
+    assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
+
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
@@ -146,14 +236,14 @@
     someinst = lambda cls, **kw: SomeInstance(bk.getuniqueclassdef(cls), **kw)
     s_inst = someinst(Exception)
     s_exc = bk.new_exception([ValueError, IndexError])
-    assert unionof(s_exc, s_inst) == s_inst
-    assert unionof(s_inst, s_exc) == s_inst
-    s_nullable = unionof(s_None, bk.new_exception([ValueError]))
+    assert union(s_exc, s_inst) == s_inst
+    assert union(s_inst, s_exc) == s_inst
+    s_nullable = union(s_None, bk.new_exception([ValueError]))
     assert isinstance(s_nullable, SomeInstance)
     assert s_nullable.can_be_None
     s_exc1 = bk.new_exception([ValueError])
     s_exc2 = bk.new_exception([IndexError])
-    unionof(s_exc1, s_exc2) == unionof(s_exc2, s_exc1)
+    union(s_exc1, s_exc2) == union(s_exc2, s_exc1)
 
 def contains_s(s_a, s_b):
     if s_b is None:
diff --git a/rpython/rtyper/test/test_llannotation.py b/rpython/rtyper/test/test_llannotation.py
--- a/rpython/rtyper/test/test_llannotation.py
+++ b/rpython/rtyper/test/test_llannotation.py
@@ -1,6 +1,6 @@
 import py.test
 from rpython.annotator.model import (
-    SomeInteger, SomeBool, SomeChar, unionof, SomeImpossibleValue,
+    SomeInteger, SomeBool, SomeChar, union, SomeImpossibleValue,
     UnionError, SomeInstance, SomeSingleFloat)
 from rpython.rlib.rarithmetic import r_uint, r_singlefloat
 from rpython.rtyper.llannotation import (
@@ -69,22 +69,22 @@
     PA1 = lltype.Ptr(lltype.GcArray())
     PA2 = lltype.Ptr(lltype.GcArray())
 
-    assert unionof(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
-    assert unionof(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS2)
+    assert union(SomePtr(PS1), SomePtr(PS2)) == SomePtr(PS1)
 
-    assert unionof(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
-    assert unionof(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA1)) == SomePtr(PA1)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA2)
+    assert union(SomePtr(PA1), SomePtr(PA2)) == SomePtr(PA1)
 
-    assert unionof(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
-    assert unionof(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
+    assert union(SomePtr(PS1), SomeImpossibleValue()) == SomePtr(PS1)
+    assert union(SomeImpossibleValue(), SomePtr(PS1)) == SomePtr(PS1)
 
     with py.test.raises(UnionError):
-        unionof(SomePtr(PA1), SomePtr(PS1))
+        union(SomePtr(PA1), SomePtr(PS1))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomePtr(PS3))
+        union(SomePtr(PS1), SomePtr(PS3))
     with py.test.raises(UnionError):
-        unionof(SomePtr(PS1), SomeInteger())
+        union(SomePtr(PS1), SomeInteger())
     with py.test.raises(UnionError):
-        unionof(SomeInteger(), SomePtr(PS1))
+        union(SomeInteger(), SomePtr(PS1))

From pypy.commits at gmail.com  Sun Nov 13 10:42:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 07:42:07 -0800 (PST)
Subject: [pypy-commit] buildbot default: pom pom pom
Message-ID: <582889cf.06891c0a.fdec9.fbf5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1019:76e3df0ab876
Date: 2016-11-13 16:41 +0100
http://bitbucket.org/pypy/buildbot/changeset/76e3df0ab876/

Log:	pom pom pom

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -463,30 +463,37 @@
                 }))
 
     if pypyjit:
-        # kill this step when the transition to test_pypy_c_new has been
-        # completed
-        # "old" test_pypy_c
-        #factory.addStep(PytestCmd(
-        #    description="pypyjit tests",
-        #    command=prefix + ["python", "pypy/test_all.py",
-        #             "--pypy=pypy/goal/pypy-c",
-        #             "--resultlog=pypyjit.log",
-        #             "pypy/module/pypyjit/test"],
-        #    logfiles={'pytestLog': 'pypyjit.log'},
-        #    env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
-        #        }))
-        #
-        # "new" test_pypy_c
-        if platform == 'win32':
-            cmd = r'pypy\goal\pypy-c'
-        else:
-            cmd = 'pypy/goal/pypy-c'
+        # run test_pypy_c tests, which is a mess because it needs to be
+        # started differently on pypy2 or pypy3
         factory.addStep(PytestCmd(
             description="pypyjit tests",
-            command=prefix + ["python", "pypy/test_all.py",
-                     "--pypy=%s" % (cmd,),
-                     "--resultlog=pypyjit_new.log",
-                     "pypy/module/pypyjit/test_pypy_c"],
+            command=prefix + ["python", "-c", r"""
+import sys, subprocess
+
+if sys.platform == 'win32':
+    cmd = r'pypy\goal\pypy-c'
+else:
+    cmd = 'pypy/goal/pypy-c'
+
+g = subprocess.check_output([cmd, '-c', 'import sys;print(sys.version)'])
+if g.startswith('2.7'):
+    # PyPy 2: must run 'pypy-c py.test'
+    cmdline = [cmd, "pypy/test_all.py"]
+elif g.startswith('3'):
+    # PyPy 3: must run 'py.test --pypy=pypy-c'
+    cmdline = [sys.executable, "pypy/test_all.py", "--pypy=" + cmd]
+else:
+    raise Exception("pypy-c output: %r" % (data,))
+
+cmdline += ["--resultlog=pypyjit_new.log",
+            "pypy/module/pypyjit/test_pypy_c"]
+
+print("Running", cmdline)
+res = subprocess.call(cmdline)
+if res:
+    print("!!!!! error code ", res)
+    sys.exit(1)
+"""],
             logfiles={'pytestLog': 'pypyjit_new.log'},
             env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest),
                 }))

From pypy.commits at gmail.com  Sun Nov 13 10:49:11 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 Nov 2016 07:49:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <58288b77.212dc20a.9adc7.22ac@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88361:da24ce183ae1
Date: 2016-11-13 15:48 +0000
http://bitbucket.org/pypy/pypy/changeset/da24ce183ae1/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -10,3 +10,7 @@
 Any uncaught RPython exception (from a PyPy bug) is turned into an
 app-level SystemError.  This should improve the lot of users hitting an
 uncaught RPython error.
+
+.. branch: union-side-effects-2
+
+Try to improve the consistency of RPython annotation unions.

From pypy.commits at gmail.com  Sun Nov 13 11:16:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 08:16:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582891cb.0370c20a.98ff1.f27c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88362:b62af9c321b8
Date: 2016-11-13 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/b62af9c321b8/

Log:	hg merge default


From pypy.commits at gmail.com  Sun Nov 13 14:44:23 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 11:44:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add operror.chain_exception(),
 to add a __context__ from rpython code.
Message-ID: <5828c297.45f6c20a.47cb3.7dc1@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88363:e1ddd470a8ac
Date: 2016-11-12 20:45 +0100
http://bitbucket.org/pypy/pypy/changeset/e1ddd470a8ac/

Log:	Add operror.chain_exception(), to add a __context__ from rpython
	code.

	But this can call normalize_exception() more often. Move w_cause out
	of normalize_exception(), to not overwrite the "suppress_context"
	flag.

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -33,12 +33,10 @@
     _w_value = None
     _application_traceback = None
     _context_recorded = False
-    w_cause = None
 
-    def __init__(self, w_type, w_value, tb=None, w_cause=None):
+    def __init__(self, w_type, w_value, tb=None):
         self.setup(w_type, w_value)
         self._application_traceback = tb
-        self.w_cause = w_cause
 
     def setup(self, w_type, w_value=None):
         assert w_type is not None
@@ -215,13 +213,6 @@
                         # raise Type, X: assume X is the constructor argument
                         w_value = space.call_function(w_type, w_value)
                     w_type = self._exception_getclass(space, w_value)
-            if self.w_cause:
-                # ensure w_cause is of a valid type
-                if space.is_none(self.w_cause):
-                    pass
-                else:
-                    self._exception_getclass(space, self.w_cause, "exception causes")
-                space.setattr(w_value, space.wrap("__cause__"), self.w_cause)
             if self._application_traceback:
                 from pypy.interpreter.pytraceback import PyTraceback
                 from pypy.module.exceptions.interp_exceptions import W_BaseException
@@ -318,6 +309,17 @@
             tb.frame.mark_as_escaped()
         return tb
 
+    def set_cause(self, space, w_cause):
+        if w_cause is None:
+            return
+        # ensure w_cause is of a valid type
+        if space.is_none(w_cause):
+            pass
+        else:
+            self._exception_getclass(space, w_cause, "exception causes")
+        w_value = self.get_w_value(space)
+        space.setattr(w_value, space.wrap("__cause__"), w_cause)
+
     def set_traceback(self, traceback):
         """Set the current traceback."""
         self._application_traceback = traceback
@@ -340,15 +342,20 @@
         last = frame._exc_info_unroll(space)
         try:
             if last is not None:
-                self.normalize_exception(space)
-                w_value = self.get_w_value(space)
-                w_last = last.get_w_value(space)
-                if not space.is_w(w_value, w_last):
-                    _break_context_cycle(space, w_value, w_last)
-                    space.setattr(w_value, space.wrap('__context__'), w_last)
+                self.chain_exceptions(space, last)
         finally:
             self._context_recorded = True
 
+    def chain_exceptions(self, space, context):
+        """Attach another OperationError as __context__."""
+        self.normalize_exception(space)
+        w_value = self.get_w_value(space)
+        context.normalize_exception(space)
+        w_context = context.get_w_value(space)
+        if not space.is_w(w_value, w_context):
+            _break_context_cycle(space, w_value, w_context)
+            space.setattr(w_value, space.wrap('__context__'), w_context)
+
     # A simplified version of _PyErr_TrySetFromCause, which returns a
     # new exception of the same class, but with another error message.
     # This only works for exceptions which have just a single message,
@@ -376,8 +383,8 @@
         try:
             new_error = oefmt(space.type(w_value),
                               "%s (%T: %S)", message, w_value, w_value)
-            new_error.w_cause = w_value
             new_error.normalize_exception(space)
+            new_error.set_cause(space, w_value)
             # Copy the traceback, but it does not escape.
             new_error.set_traceback(self._application_traceback)
         except OperationError:
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -224,8 +224,8 @@
                                    consts.CO_ITERABLE_COROUTINE):
             e2 = OperationError(space.w_RuntimeError,
                                 space.wrap("%s raised StopIteration" %
-                                           self.KIND),
-                                w_cause=e.get_w_value(space))
+                                           self.KIND))
+            e2.chain_exceptions(space, e)
             e2.record_context(space, self.frame)
             raise e2
         else:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -707,8 +707,9 @@
             w_value = space.call_function(w_type)
         else:
             w_type = space.type(w_value)
-        operror = OperationError(w_type, w_value, w_cause=w_cause)
+        operror = OperationError(w_type, w_value)
         operror.normalize_exception(space)
+        operror.set_cause(space, w_cause)
         tb = space.getattr(w_value, space.wrap('__traceback__'))
         if not space.is_w(tb, space.w_None):
             operror.set_traceback(tb)
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -302,21 +302,19 @@
         with self.lock:
             if self._closed(space):
                 return
-        w_flush_exception = None
+        flush_operr = None
         try:
             space.call_method(self, "flush")
         except OperationError as e:
-            w_flush_exception = e.get_w_value(space)
+            flush_operr = e
             raise
         finally:
             with self.lock:
                 try:
                     space.call_method(self.w_raw, "close")
                 except OperationError as e:
-                    if w_flush_exception:
-                        space.setattr(e.get_w_value(space),
-                                      space.wrap('__context__'),
-                                      w_flush_exception)
+                    if flush_operr:
+                        e.chain_exceptions(space, flush_operr)
                     raise
 
     def _dealloc_warn_w(self, space, w_source):

From pypy.commits at gmail.com  Sun Nov 13 14:44:27 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 11:44:27 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Call gc.collect() multiple times to be
 sure to free the file.
Message-ID: <5828c29b.4a6ec20a.f675a.40a5@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88365:15f0dd6bf71a
Date: 2016-11-13 19:11 +0100
http://bitbucket.org/pypy/pypy/changeset/15f0dd6bf71a/

Log:	Call gc.collect() multiple times to be sure to free the file.

diff --git a/lib-python/3/test/test_sax.py b/lib-python/3/test/test_sax.py
--- a/lib-python/3/test/test_sax.py
+++ b/lib-python/3/test/test_sax.py
@@ -16,7 +16,6 @@
 from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl
 from io import BytesIO, StringIO
 import codecs
-import gc
 import os.path
 import shutil
 from test import support
@@ -180,7 +179,7 @@
             with self.assertRaises(SAXException):
                 self.check_parse(TESTFN)
             # Collect leaked file.
-            gc.collect()
+            support.gc_collect()
         with open(TESTFN, 'rb') as f:
             with self.assertRaises(SAXException):
                 self.check_parse(f)

From pypy.commits at gmail.com  Sun Nov 13 14:44:28 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 11:44:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Refresh libmpdec source code from CPython
 3.5.1
Message-ID: <5828c29c.572d1c0a.98733.e552@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88366:ab3f12749042
Date: 2016-11-13 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/ab3f12749042/

Log:	Refresh libmpdec source code from CPython 3.5.1

diff --git a/lib_pypy/_libmpdec/io.c b/lib_pypy/_libmpdec/io.c
--- a/lib_pypy/_libmpdec/io.c
+++ b/lib_pypy/_libmpdec/io.c
@@ -446,7 +446,7 @@
 
     if (mpd_isspecial(dec)) {
 
-        mem = sizeof "-Infinity";
+        mem = sizeof "-Infinity%";
         if (mpd_isnan(dec) && dec->len > 0) {
             /* diagnostic code */
             mem += dec->digits;
@@ -609,10 +609,10 @@
             *cp++ = (flags&MPD_FMT_UPPER) ? 'E' : 'e';
             cp = exp_to_string(cp, ldigits-dplace);
         }
+    }
 
-        if (flags&MPD_FMT_PERCENT) {
-            *cp++ = '%';
-        }
+    if (flags&MPD_FMT_PERCENT) {
+        *cp++ = '%';
     }
 
     assert(cp < decstring+mem);
@@ -1260,6 +1260,9 @@
             stackspec.align = '>';
             spec = &stackspec;
         }
+        if (type == '%') {
+            flags |= MPD_FMT_PERCENT;
+        }
     }
     else {
         uint32_t workstatus = 0;
diff --git a/lib_pypy/_libmpdec/mpdecimal.c b/lib_pypy/_libmpdec/mpdecimal.c
--- a/lib_pypy/_libmpdec/mpdecimal.c
+++ b/lib_pypy/_libmpdec/mpdecimal.c
@@ -43,6 +43,7 @@
 #ifdef PPRO
   #if defined(_MSC_VER)
     #include <float.h>
+    #pragma float_control(precise, on)
     #pragma fenv_access(on)
   #elif !defined(__OpenBSD__) && !defined(__NetBSD__)
     /* C99 */
diff --git a/lib_pypy/_libmpdec/mpdecimal.h b/lib_pypy/_libmpdec/mpdecimal.h
--- a/lib_pypy/_libmpdec/mpdecimal.h
+++ b/lib_pypy/_libmpdec/mpdecimal.h
@@ -108,9 +108,13 @@
 
 #define MPD_MAJOR_VERSION 2
 #define MPD_MINOR_VERSION 4
-#define MPD_MICRO_VERSION 0
+#define MPD_MICRO_VERSION 1
 
-#define MPD_VERSION "2.4.0"
+#define MPD_VERSION "2.4.1"
+
+#define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \
+                         (MPD_MINOR_VERSION << 16) | \
+                         (MPD_MICRO_VERSION <<  8))
 
 const char *mpd_version(void);
 

From pypy.commits at gmail.com  Sun Nov 13 14:44:25 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 11:44:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Some fixes in functools
Message-ID: <5828c299.c5811c0a.d69ce.1a8c@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88364:b6dc4513e9e5
Date: 2016-11-13 17:24 +0100
http://bitbucket.org/pypy/pypy/changeset/b6dc4513e9e5/

Log:	Some fixes in functools

diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py
--- a/lib-python/3/test/test_functools.py
+++ b/lib-python/3/test/test_functools.py
@@ -129,6 +129,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -229,13 +230,7 @@
                 raise IndexError
 
         f = self.partial(object)
-        if support.check_impl_detail(pypy=True):
-            # CPython fails, pypy does not :-)
-            f.__setstate__(BadSequence())
-        else:
-            self.assertRaisesRegex(SystemError,
-                    "new style getargs format but argument is not a tuple",
-                    f.__setstate__, BadSequence())
+        self.assertRaises(TypeError, f.__setstate__, BadSequence())
 
 
 class TestPartialPy(TestPartial, unittest.TestCase):
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -42,6 +42,13 @@
         self, func, args = args[0], args[1], args[2:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
+        if isinstance(func, partial):
+            args = func._args + args
+            tmpkw = func._keywords.copy()
+            tmpkw.update(keywords)
+            keywords = tmpkw
+            del tmpkw
+            func = func._func
         self._func = func
         self._args = args
         self._keywords = keywords
@@ -113,3 +120,24 @@
             self.__dict__.clear()
         else:
             self.__dict__.update(d)
+
+
+ at builtinify
+def cmp_to_key(mycmp):
+    """Convert a cmp= function into a key= function"""
+    class K(object):
+        __slots__ = ['obj']
+        def __init__(self, obj):
+            self.obj = obj
+        def __lt__(self, other):
+            return mycmp(self.obj, other.obj) < 0
+        def __gt__(self, other):
+            return mycmp(self.obj, other.obj) > 0
+        def __eq__(self, other):
+            return mycmp(self.obj, other.obj) == 0
+        def __le__(self, other):
+            return mycmp(self.obj, other.obj) <= 0
+        def __ge__(self, other):
+            return mycmp(self.obj, other.obj) >= 0
+        __hash__ = None
+    return K

From pypy.commits at gmail.com  Sun Nov 13 15:17:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 13 Nov 2016 12:17:06 -0800 (PST)
Subject: [pypy-commit] pypy default: break too-long lines, clarify PyPy.js
Message-ID: <5828ca42.0777c20a.ec165.d8b5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88367:7ec0e5fa958a
Date: 2016-11-13 21:08 +0200
http://bitbucket.org/pypy/pypy/changeset/7ec0e5fa958a/

Log:	break too-long lines, clarify PyPy.js

diff --git a/rpython/doc/examples.rst b/rpython/doc/examples.rst
--- a/rpython/doc/examples.rst
+++ b/rpython/doc/examples.rst
@@ -7,21 +7,35 @@
 
 Actively Developed:
 
-  * PyPy, Python, very complete and maintained, http://pypy.org
-  * Pydgin, CPU emulation framework, supports ARM well, jitted, active development, https://github.com/cornell-brg/pydgin
-  * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting there, in active development https://github.com/HPI-SWA-Lab/RSqueak
-  * Pixie, 'A small, fast, native lisp with "magical" powers', jitted, maintained, https://github.com/pixie-lang/pixie
-  * Monte, 'A dynamic language inspired by Python and E.' has an rpython implementation, in active development, https://github.com/monte-language/typhon
-  * Typhon, 'A virtual machine for Monte', in active development, https://github.com/monte-language/typhon
-  * Tulip, an untyped functional language, in language design mode, maintained, https://github.com/tulip-lang/tulip/
-  * Pycket, a Racket implementation, proof of concept, small language core working, a lot of primitives are missing. Slow development https://github.com/samth/pycket
-  * Lever, a dynamic language with a modifiable grammar, actively developed, https://github.com/cheery/lever
+  * PyPy, Python2 and Python3, very complete and maintained, http://pypy.org
+  * Pydgin, CPU emulation framework, supports ARM well, jitted, active
+    development, https://github.com/cornell-brg/pydgin
+  * RSqueak VM, Smalltalk, core complete, JIT working, graphics etc getting
+    there, in active development https://github.com/HPI-SWA-Lab/RSqueak
+  * Pixie, 'A small, fast, native lisp with "magical" powers', jitted,
+    maintained, https://github.com/pixie-lang/pixie
+  * Monte, 'A dynamic language inspired by Python and E.' has an rpython
+    implementation, in active development, https://github.com/monte-language/typhon
+  * Typhon, 'A virtual machine for Monte', in active development,
+    https://github.com/monte-language/typhon
+  * Tulip, an untyped functional language, in language design mode, maintained,
+     https://github.com/tulip-lang/tulip/
+  * Pycket, a Racket implementation, proof of concept, small language core
+    working, a lot of primitives are missing. Slow development 
+    https://github.com/samth/pycket
+  * Lever, a dynamic language with a modifiable grammar, actively developed,
+    https://github.com/cheery/lever
 
 Complete, functioning, but inactive
 
   * Converge 2, complete, last release version 2.1 in Feb 2015, http://convergepl.org/
-  * Pyrolog, Prolog, core complete, extensions missing, last commit in Nov 2015, http://bitbucket.org/cfbolz/pyrolog
-  * PyPy.js, adds a JavaScript backend to pypy, http://pypyjs.org
+  * Pyrolog, Prolog, core complete, extensions missing, last commit in Nov
+    2015, http://bitbucket.org/cfbolz/pyrolog
+  * PyPy.js, compiles PyPy to Javascript via emscripten_, with a custom JIT 
+    backend that emits asm.js_ code at runtime, http://pypyjs.org
+
+.. _emscripten: http://emscripten.org
+.. _asm.js: http://asmjs.org
 
 Inactive (last reviewed Sept 2015):
 

From pypy.commits at gmail.com  Sun Nov 13 15:17:08 2016
From: pypy.commits at gmail.com (mattip)
Date: Sun, 13 Nov 2016 12:17:08 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-injection: fix cut-and-paste typo,
 XXX mark the next error
Message-ID: <5828ca44.0d1a1c0a.c22a7.543d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-injection
Changeset: r88368:53bc16aea309
Date: 2016-11-13 22:16 +0200
http://bitbucket.org/pypy/pypy/changeset/53bc16aea309/

Log:	fix cut-and-paste typo, XXX mark the next error

diff --git a/pypy/module/cpyext/injection/numpy.py b/pypy/module/cpyext/injection/numpy.py
--- a/pypy/module/cpyext/injection/numpy.py
+++ b/pypy/module/cpyext/injection/numpy.py
@@ -73,14 +73,16 @@
         data1 = rffi.cast(rffi.DOUBLEP, arr1.data)
         n1 = arr1.dimensions[0]
     else:
+        # XXX this should be a pointer to a float, not a list
         data1 = [space.float_w(w_self),]
         n1 = 1
     if isinstance(w_other, W_ArrayObject):
-        arr2 = rffi.cast(PyArrayObject, w_self.pyobj)
+        arr2 = rffi.cast(PyArrayObject, w_other.pyobj)
         data2 = rffi.cast(rffi.DOUBLEP, arr2.data)
         n2 = arr2.dimensions[0]
     else:
-        data2 = [space.int_w(w_other),]
+        # XXX this should be a pointer to a float, not a list
+        data2 = [space.float_w(w_other),]
         n2 = 1
     if not (n1 == n2 or n1 == 1 or n2 == 1):
         raise oefmt(space.w_ValueError, 'dimension mismatch')

From pypy.commits at gmail.com  Sun Nov 13 17:29:44 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 14:29:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: CPython issue #21679: Prevent extraneous
 fstat() calls during open().
Message-ID: <5828e958.43e61c0a.3cdac.f79c@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88369:9ffa558330fb
Date: 2016-11-13 23:22 +0100
http://bitbucket.org/pypy/pypy/changeset/9ffa558330fb/

Log:	CPython issue #21679: Prevent extraneous fstat() calls during
	open().

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -5,9 +5,13 @@
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import rposix
+from rpython.rlib.rposix_stat import STAT_FIELD_TYPES
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
-from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
+from pypy.module._io.interp_iobase import (
+    W_RawIOBase, convert_size, DEFAULT_BUFFER_SIZE)
+
+HAS_BLKSIZE = 'st_blksize' in STAT_FIELD_TYPES
 
 def interp_member_w(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
@@ -162,12 +166,6 @@
         fd_is_own = False
         try:
             if fd >= 0:
-                try:
-                    os.fstat(fd)
-                except OSError as e:
-                    if e.errno == errno.EBADF:
-                        raise wrap_oserror(space, e)
-                    # else: pass
                 self.fd = fd
                 self.closefd = bool(closefd)
             elif space.is_none(w_opener):
@@ -207,7 +205,20 @@
                     except OSError as e:
                         raise wrap_oserror2(space, e, w_name)
 
-            self._dircheck(space, w_name)
+            try:
+                st = os.fstat(self.fd)
+            except OSError as e:
+                raise wrap_oserror(space, e)
+            # On Unix, fopen will succeed for directories.
+            # In Python, there should be no file objects referring to
+            # directories, so we need a check.
+            if stat.S_ISDIR(st.st_mode):
+                raise wrap_oserror2(space, OSError(errno.EISDIR, "fstat"),
+                                    w_name, exception_name='w_IOError')
+            self.blksize = DEFAULT_BUFFER_SIZE
+            if HAS_BLKSIZE and st.st_blksize > 1:
+                self.blksize = st.st_blksize
+
             space.setattr(self, space.wrap("name"), w_name)
 
             if self.appending:
@@ -244,6 +255,9 @@
     def descr_get_mode(self, space):
         return space.wrap(self._mode())
 
+    def get_blksize(self, space):
+        return space.wrap(self.blksize)
+
     def _closed(self, space):
         return self.fd < 0
 
@@ -298,20 +312,6 @@
                 if e.match(space, space.w_Warning):
                     e.write_unraisable(space, '', space.wrap(self))
 
-    def _dircheck(self, space, w_filename):
-        # On Unix, fopen will succeed for directories.
-        # In Python, there should be no file objects referring to
-        # directories, so we need a check.
-        if self.fd < 0:
-            return
-        try:
-            st = os.fstat(self.fd)
-        except OSError:
-            return
-        if stat.S_ISDIR(st.st_mode):
-            raise wrap_oserror2(space, OSError(errno.EISDIR, "fstat"),
-                                w_filename, exception_name='w_IOError')
-
     @unwrap_spec(pos=r_longlong, whence=int)
     def seek_w(self, space, pos, whence=0):
         self._check_closed(space)
@@ -506,5 +506,6 @@
         doc="True if the file descriptor will be closed"),
     mode = GetSetProperty(W_FileIO.descr_get_mode,
                           doc="String giving the file mode"),
+    _blksize = GetSetProperty(W_FileIO.get_blksize),
     )
 
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -6,9 +6,6 @@
     TypeDef, interp_attrproperty, generic_new_descr)
 from pypy.module._io.interp_fileio import W_FileIO
 from pypy.module._io.interp_textio import W_TextIOWrapper
-from rpython.rlib.rposix_stat import STAT_FIELD_TYPES
-
-HAS_BLKSIZE = 'st_blksize' in STAT_FIELD_TYPES
 
 
 class Cache:
@@ -17,8 +14,6 @@
             "io.UnsupportedOperation",
             space.newtuple([space.w_ValueError, space.w_IOError]))
 
-DEFAULT_BUFFER_SIZE = 8 * 1024
-
 @unwrap_spec(mode=str, buffering=int,
              encoding="str_or_None", errors="str_or_None",
              newline="str_or_None", closefd=int)
@@ -96,18 +91,7 @@
         buffering = -1
 
     if buffering < 0:
-        buffering = DEFAULT_BUFFER_SIZE
-
-        if HAS_BLKSIZE:
-            fileno = space.c_int_w(space.call_method(w_raw, "fileno"))
-            try:
-                st = os.fstat(fileno)
-            except OSError:
-                # Errors should never pass silently, except this one time.
-                pass
-            else:
-                if st.st_blksize > 1:
-                    buffering = st.st_blksize
+        buffering = space.c_int_w(space.getattr(w_raw, space.wrap("_blksize")))
 
     if buffering < 0:
         raise oefmt(space.w_ValueError, "invalid buffering size")
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -21,6 +21,8 @@
         assert f.name.endswith('tmpfile')
         assert f.mode == 'ab'
         assert f.closefd is True
+        assert f._blksize >= 1024
+        assert f._blksize % 1024 == 0
         f.close()
 
     def test_invalid_fd(self):

From pypy.commits at gmail.com  Sun Nov 13 17:29:46 2016
From: pypy.commits at gmail.com (amauryfa)
Date: Sun, 13 Nov 2016 14:29:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add 'closefd=True/False' to
 FileIO.__repr__
Message-ID: <5828e95a.e626c20a.c4c66.6f80@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r88370:24e0383cef36
Date: 2016-11-13 23:28 +0100
http://bitbucket.org/pypy/pypy/changeset/24e0383cef36/

Log:	Add 'closefd=True/False' to FileIO.__repr__

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -368,15 +368,17 @@
         if self.fd < 0:
             return space.wrap("<_io.FileIO [closed]>")
 
+        closefd = "True" if self.closefd else "False"
+
         if self.w_name is None:
             return space.wrap(
-                "<_io.FileIO fd=%d mode='%s'>" % (
-                    self.fd, self._mode()))
+                "<_io.FileIO fd=%d mode='%s' closefd=%s>" % (
+                    self.fd, self._mode(), closefd))
         else:
             w_repr = space.repr(self.w_name)
             return space.wrap(
-                "<_io.FileIO name=%s mode='%s'>" % (
-                    space.str_w(w_repr), self._mode()))
+                "<_io.FileIO name=%s mode='%s' closefd=%s>" % (
+                    space.str_w(w_repr), self._mode(), closefd))
 
     # ______________________________________________
 
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -188,10 +188,10 @@
     def test_repr(self):
         import _io
         f = _io.FileIO(self.tmpfile, 'r')
-        assert repr(f) == ("<_io.FileIO name=%r mode='%s'>"
+        assert repr(f) == ("<_io.FileIO name=%r mode='%s' closefd=True>"
                            % (f.name, f.mode))
         del f.name
-        assert repr(f) == ("<_io.FileIO fd=%r mode='%s'>"
+        assert repr(f) == ("<_io.FileIO fd=%r mode='%s' closefd=True>"
                            % (f.fileno(), f.mode))
         f.close()
         assert repr(f) == "<_io.FileIO [closed]>"

From pypy.commits at gmail.com  Mon Nov 14 02:47:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 Nov 2016 23:47:23 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: updates on pypy3.3/pypy3.5
Message-ID: <58296c0b.448e1c0a.ecbe1.9441@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r820:2f49dd4737a5
Date: 2016-11-14 08:47 +0100
http://bitbucket.org/pypy/pypy.org/changeset/2f49dd4737a5/

Log:	updates on pypy3.3/pypy3.5

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -75,7 +75,7 @@
 <p>We provide binaries for x86, ARM, and PPC Linux, Mac OS/X and Windows for:</p>
 <ul class="simple">
 <li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.6.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy2.7-v5.6.0.html">what's new in PyPy2.7?</a>)</li>
-<li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.5-alpha</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html">what's new in PyPy3.3?</a>).</li>
+<li>the Python3.3 compatible release &#8212; <strong>PyPy3.3 v5.5</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-pypy3.3-v5.5-alpha.html">what's new in PyPy3.3?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
 <ul class="download-menu simple">
@@ -136,8 +136,11 @@
 </div>
 <div class="section" id="python-3-3-5-compatible-pypy3-3-v5-5">
 <h1>Python 3.3.5 compatible PyPy3.3 v5.5</h1>
-<p class="download-menu">Warning: this is an alpha release supporting the Python 3.3 language.
-It's also known to be (sometimes much) slower than PyPy 2.</p>
+<p class="download-menu">Warning: PyPy3.3 is considered <strong>alpha/beta software.</strong>  All binaries
+are thus called &ldquo;alpha&rdquo;.  It is known to be sometimes much slower than
+PyPy 2.  You are welcome to use it anyway; if you're lucky it will
+be fast in your case.  We are currently working on <a class="reference external" href="http://buildbot.pypy.org/nightly/py3.5/">PyPy3.5 supporting
+Python 3.5.x</a>, but it is not ready for release yet.</p>
 <ul class="simple">
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -16,7 +16,7 @@
 
 * the Python2.7 compatible release &#8212; **PyPy2.7 v5.6.0** &#8212; (`what's new in PyPy2.7?`_)
 
-* the Python3.3 compatible release &#8212; **PyPy3.3 v5.5-alpha** &#8212; (`what's new in PyPy3.3?`_).
+* the Python3.3 compatible release &#8212; **PyPy3.3 v5.5** &#8212; (`what's new in PyPy3.3?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
@@ -117,8 +117,13 @@
 
 .. class:: download_menu
 
-    Warning: this is an alpha release supporting the Python 3.3 language.
-    It's also known to be (sometimes much) slower than PyPy 2.
+    Warning: PyPy3.3 is considered **alpha/beta software.**  All binaries
+    are thus called "alpha".  It is known to be sometimes much slower than
+    PyPy 2.  You are welcome to use it anyway; if you're lucky it will
+    be fast in your case.  We are currently working on `PyPy3.5 supporting
+    Python 3.5.x`__, but it is not ready for release yet.
+
+.. __: http://buildbot.pypy.org/nightly/py3.5/
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
 * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)

From pypy.commits at gmail.com  Mon Nov 14 06:05:59 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 Nov 2016 03:05:59 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges: merge default
Message-ID: <58299a97.e576c20a.f87aa.3616@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges
Changeset: r88371:b0c428c5ac48
Date: 2016-11-13 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/b0c428c5ac48/

Log:	merge default

diff too long, truncating to 2000 out of 15945 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -2,6 +2,7 @@
 from ctypes.test import need_symbol, xfail
 import unittest
 import os
+import sys
 
 import ctypes
 import _ctypes_test
@@ -279,7 +280,6 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
 
-    @xfail
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
         # Issue #23319
@@ -295,5 +295,11 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
 
+    # see issue #1213, on big endian it fails for the little endian case
+    if sys.byteorder == 'little':
+        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
+    elif sys.byteorder == 'big':
+        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -29,7 +29,7 @@
 class deque(object):
 
     def __new__(cls, iterable=(), *args, **kw):
-        self = super(deque, cls).__new__(cls, *args, **kw)
+        self = super(deque, cls).__new__(cls)
         self.clear()
         return self
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -229,7 +229,7 @@
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
-        self = super(_CData, cls).__new__(cls, *args, **kwds)
+        self = super(_CData, cls).__new__(cls)
         if '_abstract_' in cls.__dict__:
             raise TypeError("abstract class")
         if hasattr(cls, '_ffistruct_'):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -334,6 +334,8 @@
                         realtype, quals = self._get_type_and_quals(
                             decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
                 else:
                     raise api.CDefError("unrecognized construct", decl)
         except api.FFIError as e:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,10 +519,18 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            raise api.CDefError("%r has no values explicitly defined: "
-                                "refusing to guess which integer type it is "
-                                "meant to be (unsigned/signed, int/long)"
-                % self._get_c_name())
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -1,4 +1,5 @@
 import os
+import sys
 
 try:
     basestring
@@ -74,8 +75,13 @@
     Add py_limited_api to kwds if setuptools >= 26 is in use.
     Do not alter the setting if it already exists.
     Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+
+    CPython itself should ignore the flag in a debugging version
+    (by not listing .abi3.so in the extensions it supports), but
+    it doesn't so far, creating troubles.  That's why we check
+    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds:
+    if 'py_limited_api' not in kwds and not sys.flags.debug:
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -9,8 +9,8 @@
   Philip Jenvey
   Ronan Lamy
   Brian Kearns
+  Richard Plangger
   Michael Hudson
-  Richard Plangger
   Manuel Jacob
   David Schneider
   Holger Krekel
@@ -54,10 +54,10 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
-  Spenser Bauman
   Tyler Wade
   Vincent Legoll
   Michael Foord
@@ -65,13 +65,13 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Stefano Rivera
   Timo Paulssen
   Edd Barrett
   Squeaky
@@ -89,12 +89,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -128,10 +128,11 @@
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -147,17 +148,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
   Daniel Patrick
-  Jasper Schulz
-  Christian Hudon
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -165,8 +165,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -175,11 +175,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -196,7 +197,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -219,6 +219,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -231,6 +232,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -258,9 +260,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -274,10 +274,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
-  "Aaron Gallagher
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -287,8 +287,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -302,13 +303,11 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
-  JohnDoe
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
   reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
@@ -316,25 +315,24 @@
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
   remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
   werat
-  hgattic
+
+
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -397,3 +397,28 @@
 in auto-generated C code, and at least some knowledge about the
 various components involved, from PyPy's own RPython source code to
 the GC and possibly the JIT.
+
+
+Why doesn't PyPy move to GitHub, Gitlab, ...?
+----------------------------------------------
+
+We've been quite happy with bitbucket.org. Moving version control systems and
+hosting is a lot of hard work: On the one hand, PyPy's mercurial history is
+long and gnarly. On the other hand, all our infrastructure (buildbots,
+benchmarking, etc) would have to be adapted. So unless somebody steps up and
+volunteers to do all that work, it will likely not happen.
+
+
+What is needed for Windows 64 support of PyPy?
+-----------------------------------------------
+
+First, please note that the Windows 32 PyPy binary works just fine on Windows
+64. The only problem is that it only supports up to 4GB of heap per process.
+
+As to real Windows 64 support: Currently we don't have an active PyPy developer
+whose main development platform is Windows. So if you are interested in getting
+Windows 64 support, we encourage you to volunteer `to make it happen`_! Another
+option would be to pay some PyPy developers to implement Windows 64 support,
+but so far there doesn't seem to be an overwhelming commercial interest in it.
+
+.. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -1,4 +1,4 @@
-The PyPy Release Process
+PyPy's Release Process
 ========================
 
 Release Policy
@@ -39,41 +39,60 @@
   Some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
   necessary.
-* update pypy/doc/contributor.rst (and possibly LICENSE)
-  pypy/doc/tool/makecontributor.py generates the list of contributors
-* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  create a fresh whatsnew_head.rst after the release
-  and add the new file to  pypy/doc/index-of-whatsnew.rst
-* go to pypy/tool/release and run
-  ``force-builds.py <release branch>``
-  The following JIT binaries should be built, however, we need more buildbots
-  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
-  freebsd64 
 
-* wait for builds to complete, make sure there are no failures
-* download the builds, repackage binaries. Tag the release version
-  and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+* Update and write documentation
 
-  Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
-  and to cobra, as some packagers prefer a clearly labeled source package
-  ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
-  unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+  * update pypy/doc/contributor.rst (and possibly LICENSE)
+    pypy/doc/tool/makecontributor.py generates the list of contributors
 
-* Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+  * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+    create a fresh whatsnew_head.rst after the release
+    and add the new file to  pypy/doc/index-of-whatsnew.rst
 
-* write release announcement pypy/doc/release-x.y(.z).txt
+  * write release announcement pypy/doc/release-VERSION.rst
+    The release announcement should contain a direct link to the download page
 
-  The release announcement should contain a direct link to the download page
+  * Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
 
-* Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
+* Build and upload the release tar-balls
 
-* update pypy.org (under extradoc/pypy.org), rebuild and commit
+  * go to pypy/tool/release and run
+    ``force-builds.py <release branch>``
+    The following JIT binaries should be built, however, we need more buildbots
+    windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+    freebsd64 
 
-* post announcement on morepypy.blogspot.com
-* send announcements to twitter.com, pypy-dev, python-list,
-  python-announce, python-dev ...
+  * wait for builds to complete, make sure there are no failures
 
-* add a tag on the pypy/jitviewer repo that corresponds to pypy release
-* add a tag on the codespeed web site that corresponds to pypy release
-* revise versioning at https://readthedocs.org/projects/pypy
+  * send out a mailing list message asking for people to test before uploading
+    to prevent having to upload more than once
+
+  * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
+    that the source tarball can be produced in the next steps
+
+  * download the builds, repackage binaries. Tag the release version
+    and download and repackage source from bitbucket. You may find it
+    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+
+    Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
+    and to cobra, as some packagers prefer a clearly labeled source package
+    ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
+    unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
+  * Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+
+* Send out a mailing list message asking for last-minute comments and testing
+
+* RELEASE !  
+
+  * update pypy.org (under extradoc/pypy.org), rebuild and commit, using the
+    hashes produced from the ``repackage.sh`` script or by hand
+
+  * post announcement on morepypy.blogspot.com
+  * send announcements to twitter.com, pypy-dev, python-list,
+    python-announce, python-dev ...
+
+* If all is OK, document the released version
+
+  * add a tag on the codespeed web site that corresponds to pypy release
+  * revise versioning at https://readthedocs.org/projects/pypy
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -1,8 +1,17 @@
 Downloading and Installing PyPy
 ===============================
 
+Using a packaged PyPy
+~~~~~~~~~~~~~~~~~~~~~
+
+Some Linux distributions provide a pypy package. Note that in order to
+install additional modules that require compilation, you may need to install
+additional packages such as pypy-dev. This will manifest as an error about
+"missing Python.h". Distributions do not as of yet supply many pypy-ready
+packages, if you require additional modules we recommend creating a virtualenv
+and using pip. 
+
 .. _prebuilt-pypy:
-
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -38,6 +47,9 @@
 and not move the binary there, else PyPy would not be able to find its
 library.
 
+Installing more modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
 If you want to install 3rd party libraries, the most convenient way is
 to install pip_ using ensurepip_ (unless you want to install virtualenv as 
 explained below; then you can directly use pip inside virtualenvs):
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -0,0 +1,155 @@
+============
+PyPy2.7 v5.6
+============
+
+We have released PyPy2.7 v5.6, about two months after PyPy2.7 v5.4.
+This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
+
+We continue to make incremental improvements to our C-API
+compatibility layer (cpyext). We pass all but a few of the tests in the
+upstream numpy `test suite`_. 
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, and some of those
+changes have been backported to PyPy2.7 where relevant.
+
+The PowerPC and s390x backend have been enhanced_ with the capability use SIMD instructions
+ for micronumpy loops.
+
+We changed ``timeit`` to now report average +- standard deviation, which is
+better than the misleading minimum value reported in CPython.
+
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
+well as maintaining support for previous versions.
+
+CFFI_ has been updated to 1.9, improving an already great package for
+interfacing with C.
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the PyPy2.7 v5.6 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: http://doc.pypy.org
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+.. _`enhanced`: https://morepypy.blogspot.co.at/2016/11/vectorization-extended-powerpc-and-s390x.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Other Highlights (since 5.4 released Aug 31, 2016)
+=========================================================
+
+* New features
+
+  * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+    dynamically loadable file
+  * Backport fixes to rposix on windows from py3.5
+  * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
+  * Add ``inode`` to ``scandir()`` on posix systems
+  * Support more attributes on ``super``
+  * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
+  * Restore the ability to translate with CPython
+  * Update to CFFI 1.9.0
+  * Support the new buffer protocol in cpyext and numpypy
+  * Add ``rposix.sync()``
+  * Support full-precision nanosecond times in os.stat()
+  * Add documentation about the assembler backends to RPYthon
+  * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
+    changes downstream packaging requirements
+  * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
+  * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
+    be ``str`` or ``unicode``)
+  * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
+  * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
+    relevant platforms
+  * Backport the ``faulthandler`` module from py3.5
+  * Improve the error message when trying to call a method where the ``self``
+    parameter is missing in the definition
+  * Implement ``rposix.cpu_count``
+  * Support translation on FreeBSD running on PowerPC
+  * Implement ``__rmod__`` on ``str`` and ``unicode`` types
+  * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
+
+* Bug Fixes
+
+  * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+  * Fix translation of the sandbox
+  * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
+    the last byte of a unicode string sometimes
+  * fix some calls to functions through window's COM interface
+  * fix minor leak when the C call to socketpair() fails
+  * make sure (-1.0 + 0j).__hash__(), (-1.0).__hash__() returns -2
+  * Fix for an issue where PyBytesResize was called on a fresh pyobj
+  * Fix bug in codewriter about passing the ``exitswitch`` variable to a call
+  * Don't crash in ``merge_if_blocks`` if the values are symbolics
+  * Issue #2325/2361: __class__ assignment between two classes with the same
+    slots
+  * Issue #2409: don't leak the file descriptor when doing ``open('some-dir')``
+  * Windows fixes around vmprof
+  * Don't use ``sprintf()`` from inside a signal handler
+  * Test and fix bug from the ``guard_not_forced_2`` branch, which didn't
+    save the floating-point register
+  * ``_numpypy.add.reduce`` returns a scalar now
+
+* Performance improvements:
+
+  * Improve method calls on oldstyle classes
+  * Clean and refactor code for testing cpyext to allow sharing with py3.5
+  * Refactor a building the map of reflected ops in ``_numpypy``
+  * Improve merging of virtual states in the JIT in order to avoid jumping to the
+    preamble
+  * In JIT residual calls, if the called function starts with a fast-path like
+    ``if x.foo != 0: return x.foo``, then inline the check before doing the
+     ``CALL``.
+  * Ensure ``make_inputargs`` fails properly when given arguments with type 
+    information
+  * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
+    about more easily and debugging is faster
+  * Refactor and remove dead code from ``optimizeopt``, ``resume``
+  
+
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-5.6.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -110,4 +110,4 @@
         "hg log -r %s --template '{branch}'" % startrev)
     if errcode != 0:
         py.test.skip('no Mercurial repo')
-    assert wc_branch == 'default'
+    assert wc_branch in ('default', "'default'") # sometimes the ' leaks (windows)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -32,7 +32,7 @@
     'Niklaus Haldimann': ['nik'],
     'Alexander Schremmer': ['xoraxax'],
     'Anders Hammarquist': ['iko'],
-    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'David Edelsohn': ['edelsoh', 'edelsohn','opassembler.py'],
     'Niko Matsakis': ['niko'],
     'Jakub Gustak': ['jlg'],
     'Guido Wesdorp': ['guido'],
@@ -75,6 +75,9 @@
     'Spenser Bauman':['Spenser Andrew Bauman'],
     'Raffael Tfirst':['raffael.tfirst at gmail.com'],
     'timo':['timo at eistee.fritz.box'],
+    'Jasper Schulz':['Jasper.Schulz'],
+    'Aaron Gallagher':['"Aaron Gallagher'],
+    'Yasir Suhail':['yasirs'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,82 +1,12 @@
 ==========================
-What's new in PyPy2.7 5.4+
+What's new in PyPy2.7 5.6+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 522736f816dc
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
 
-.. branch: rpython-resync
-Backport rpython changes made directly on the py3k and py3.5 branches.
+.. branch: rpython-error-to-systemerror
 
-.. branch: buffer-interface
-Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
-in numpypy
-
-.. branch: force-virtual-state
-Improve merging of virtual states in the JIT in order to avoid jumping to the
-preamble. Accomplished by allocating virtual objects where non-virtuals are
-expected.
-
-.. branch: conditional_call_value_3
-JIT residual calls: if the called function starts with a fast-path
-like "if x.foo != 0: return x.foo", then inline the check before
-doing the CALL.  For now, string hashing is about the only case.
-
-.. branch: search-path-from-libpypy
-
-The compiled pypy now looks for its lib-python/lib_pypy path starting
-from the location of the *libpypy-c* instead of the executable. This is
-arguably more consistent, and also it is what occurs anyway if you're
-embedding pypy.  Linux distribution packagers, take note!  At a minimum,
-the ``libpypy-c.so`` must really be inside the path containing
-``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
-from somewhere else.  You no longer have to do the same with the
-``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
-
-.. branch: _warnings
-
-CPython allows warning.warn(('something', 1), Warning), on PyPy this
-produced a "expected a readable buffer object" error. Test and fix.
-
-.. branch: stricter-strip
-
-CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
-allowed as arguments. Test and fix for str and unicode
-
-.. branch: faulthandler
-
-Port the 'faulthandler' module to PyPy default.  This module is standard
-in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
-
-.. branch: test-cpyext
-
-Refactor cpyext testing to be more pypy3-friendly.
-
-.. branch: better-error-missing-self
-
-Improve the error message when the user forgot the "self" argument of a method.
-
-
-.. fb6bb835369e
-Change the ``timeit`` module: it now prints the average time and the standard
-deviation over 7 runs by default, instead of the minimum. The minimum is often
-misleading.
-
-.. branch: unrecursive-opt
-
-Make optimiseopt iterative instead of recursive so it can be reasoned about
-more easily and debugging is faster.
-
-.. branch: Tiberiumk/fix-2412-1476011166874
-.. branch: redirect-assembler-jitlog
-
-
-
-.. branch: stdlib-2.7.12
-
-Update stdlib to version 2.7.12
-
-.. branch: buffer-interface2
-
-Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
-types still missing
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.4+
-==========================
+=========================
+What's new in PyPy2.7 5.6
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.4
 .. startrev: 522736f816dc
@@ -80,3 +80,28 @@
 
 Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
 types still missing
+
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: zarch-simd-support
+
+s390x implementation for vector operations used in VecOpt
+
+.. branch: ppc-vsx-support
+
+PowerPC implementation for vector operations used in VecOpt
+
+.. branch: newinitwarn
+
+Match CPython's stricter handling of __new/init__ arguments
+
+.. branch: openssl-1.1
+
+Support for OpenSSL version 1.1 (in addition to version 1.0).
+Tested on Linux (1.1, 1.0), on Win32, and Mac (1.0 only)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -20,6 +20,22 @@
 
 .. _our downloads: http://pypy.org/download.html
 
+Installing Visual Compiler v9 (for Python 2.7)
+----------------------------------------------
+
+This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
+was checked in Nov 2016). Note that the compiler suite will be installed in
+``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
+Using a current version of ``setuptools`` will be able to find it there. For
+Windows 10, you must right-click the download, and under ``Properties`` ->
+``Compatibility`` mark it as ``Run run this program in comatibility mode for``
+``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
+otherwise ``mt.exe`` will silently fail. Installation will begin automatically
+by running the mt.exe command by hand from a DOS window (that is how the author
+discovered the problem).
+
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
 
 Translating PyPy with Visual Studio
 -----------------------------------
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -33,7 +33,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -103,18 +108,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -122,6 +132,9 @@
                 debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
@@ -240,7 +253,8 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if config.translation.profopt is not None:
+            if (config.translation.profopt is not None
+                    and not config.translation.noprofopt):
                 raise Exception("Cannot use the --profopt option "
                                 "when --shared is on (it is by default). "
                                 "See issue #2398.")
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -275,6 +275,10 @@
 def set_fully_buffered_io():
     sys.stdout = sys.__stdout__ = fdopen(1, 'w')
 
+def initstdio(unbuffered=False):
+    if unbuffered:
+        set_unbuffered_io()
+
 # ____________________________________________________________
 # Main entry point
 
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,6 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib import rstackovf
 
 from pypy.interpreter import debug
 
@@ -58,10 +59,14 @@
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
         s = self._w_value
-        if self.__class__ is not OperationError and s is None:
-            space = getattr(self.w_type, 'space')
-            if space is not None:
+        space = getattr(self.w_type, 'space', None)
+        if space is not None:
+            if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
+            try:
+                s = space.str_w(s)
+            except Exception:
+                pass
         return '[%s: %s]' % (self.w_type, s)
 
     def errorstr(self, space, use_repr=False):
@@ -73,11 +78,8 @@
             exc_value = str(w_value)
         else:
             w = space.wrap
-            if space.is_w(space.type(self.w_type), space.w_str):
-                exc_typename = space.str_w(self.w_type)
-            else:
-                exc_typename = space.str_w(
-                    space.getattr(self.w_type, w('__name__')))
+            exc_typename = space.str_w(
+                space.getattr(self.w_type, w('__name__')))
             if space.is_w(w_value, space.w_None):
                 exc_value = ""
             else:
@@ -508,3 +510,47 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+ at jit.dont_look_inside
+def get_converted_unexpected_exception(space, e):
+    """This is used in two places when we get an non-OperationError
+    RPython exception: from gateway.py when calling an interp-level
+    function raises; and from pyopcode.py when we're exiting the
+    interpretation of the frame with an exception.  Note that it
+    *cannot* be used in pyopcode.py: that place gets a
+    ContinueRunningNormally exception from the JIT, which must not end
+    up here!
+    """
+    try:
+        if not we_are_translated():
+            raise
+        raise e
+    except KeyboardInterrupt:
+        return OperationError(space.w_KeyboardInterrupt, space.w_None)
+    except MemoryError:
+        return OperationError(space.w_MemoryError, space.w_None)
+    except rstackovf.StackOverflow as e:
+        # xxx twisted logic which happens to give the result that we
+        # want: when untranslated, a RuntimeError or its subclass
+        # NotImplementedError is caught here.  Then
+        # check_stack_overflow() will re-raise it directly.  We see
+        # the result as this exception propagates directly.  But when
+        # translated, an RPython-level RuntimeError is turned into
+        # an app-level RuntimeError by the next case.
+        rstackovf.check_stack_overflow()
+        return oefmt(space.w_RuntimeError,
+                     "maximum recursion depth exceeded")
+    except RuntimeError:   # not on top of py.py
+        return OperationError(space.w_RuntimeError, space.w_None)
+    except:
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
+        else:
+            # when untranslated, we don't wrap into an app-level
+            # SystemError (this makes debugging tests harder)
+            raise
+        return OperationError(space.w_SystemError, space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -611,7 +611,7 @@
 
 class StaticMethod(W_Root):
     """The staticmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -622,13 +622,16 @@
 
     def descr_staticmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(StaticMethod, w_subtype)
-        instance.__init__(w_function)
-        return space.wrap(instance)
+        instance.__init__(space.w_None)
+        return instance
+
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
 
 
 class ClassMethod(W_Root):
     """The classmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -641,8 +644,11 @@
 
     def descr_classmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(ClassMethod, w_subtype)
-        instance.__init__(w_function)
-        return space.wrap(instance)
+        instance.__init__(space.w_None)
+        return instance
+
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
 
 class FunctionWithFixedCode(Function):
     can_change_code = False
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -23,7 +23,6 @@
     DescrMismatch)
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode
-from rpython.rlib import rstackovf
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
@@ -712,16 +711,11 @@
             if not we_are_translated():
                 raise
             raise e
-        except KeyboardInterrupt:
-            raise OperationError(space.w_KeyboardInterrupt, space.w_None)
-        except MemoryError:
-            raise OperationError(space.w_MemoryError, space.w_None)
-        except rstackovf.StackOverflow as e:
-            rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RuntimeError,
-                        "maximum recursion depth exceeded")
-        except RuntimeError:   # not on top of py.py
-            raise OperationError(space.w_RuntimeError, space.w_None)
+        except OperationError:
+            raise
+        except Exception as e:      # general fall-back
+            from pypy.interpreter import error
+            raise error.get_converted_unexpected_exception(space, e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,6 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+import sys
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
@@ -278,6 +279,10 @@
                         self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
+            except OperationError:
+                raise
+            except Exception as e:      # general fall-back
+                raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
@@ -883,6 +888,14 @@
             frame = frame.f_backref()
         return None
 
+    def _convert_unexpected_exception(self, e):
+        from pypy.interpreter import error
+
+        operr = error.get_converted_unexpected_exception(self.space, e)
+        pytraceback.record_application_traceback(
+            self.space, operr, self, self.last_instr)
+        raise operr
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -1,8 +1,10 @@
 import py
 from pypy.interpreter import executioncontext
+from pypy.interpreter.error import OperationError
 
-class Finished(Exception):
-    pass
+class Finished(OperationError):
+    def __init__(self):
+        OperationError.__init__(self, "exception_class", "exception_value")
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -4,6 +4,7 @@
 from pypy.interpreter import gateway, argument
 from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.error import OperationError
 import py
 import sys
 
@@ -771,6 +772,42 @@
         w_g = space.wrap(gateway.interp2app_temp(g, doc='bar'))
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
+    def test_system_error(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        e = py.test.raises(OperationError, space.appexec, [w_g], """(my_g):
+            my_g()
+        """)
+        err = str(e.value)
+        assert 'SystemError' in err
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
+    def test_system_error_2(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_msg = space.appexec([w_g], """(my_g):
+            try:
+                my_g()
+            except SystemError as e:
+                return str(e)
+        """)
+        err = space.str_w(w_msg)
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -20,7 +20,7 @@
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    lls = rffi.str2charp("sys")
+    lls = rffi.str2charp("sys   # should give a NameError")
     execute_source(lls)
     lltype.free(lls, flavor='raw')
     # did not crash - the same globals
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
@@ -686,15 +688,17 @@
 (e.g. C().f()).  The instance is ignored except for its class.""",
     __get__ = interp2app(StaticMethod.descr_staticmethod_get),
     __new__ = interp2app(StaticMethod.descr_staticmethod__new__.im_func),
+    __init__=interp2app(StaticMethod.descr_init),
     __func__= interp_attrproperty_w('w_function', cls=StaticMethod),
     )
 
 ClassMethod.typedef = TypeDef(
     'classmethod',
-    __new__ = interp2app(ClassMethod.descr_classmethod__new__.im_func),
-    __get__ = interp2app(ClassMethod.descr_classmethod_get),
-    __func__= interp_attrproperty_w('w_function', cls=ClassMethod),
-    __doc__ = """classmethod(function) -> class method
+    __new__=interp2app(ClassMethod.descr_classmethod__new__.im_func),
+    __init__=interp2app(ClassMethod.descr_init),
+    __get__=interp2app(ClassMethod.descr_classmethod_get),
+    __func__=interp_attrproperty_w('w_function', cls=ClassMethod),
+    __doc__="""classmethod(function) -> class method
 
 Convert a function to be a class method.
 
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -509,59 +509,6 @@
         assert eval("i", None, None) == 4
         assert eval('a', None, dict(a=42)) == 42
 
-    def test_compile(self):
-        co = compile('1+2', '?', 'eval')
-        assert eval(co) == 3
-        co = compile(buffer('1+2'), '?', 'eval')
-        assert eval(co) == 3
-        exc = raises(TypeError, compile, chr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
-        compile("from __future__ import with_statement", "<test>", "exec")
-        raises(SyntaxError, compile, '-', '?', 'eval')
-        raises(ValueError, compile, '"\\xt"', '?', 'eval')
-        raises(ValueError, compile, '1+2', '?', 'maybenot')
-        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
-        raises(TypeError, compile, '1+2', 12, 34)
-
-    def test_compile_error_message(self):
-        import re
-        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
-        exc = raises(SyntaxError, compile,
-            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        assert 'iso-8859-15' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-
-    def test_unicode_compile(self):
-        try:
-            compile(u'-', '?', 'eval')
-        except SyntaxError as e:
-            assert e.lineno == 1
-
-    def test_unicode_encoding_compile(self):
-        code = u"# -*- coding: utf-8 -*-\npass\n"
-        raises(SyntaxError, compile, code, "tmp", "exec")
-
-    def test_recompile_ast(self):
-        import _ast
-        # raise exception when node type doesn't match with compile mode
-        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
-        raises(TypeError, compile, co1, '<ast>', 'eval')
-        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
-
     def test_isinstance(self):
         assert isinstance(5, int)
         assert isinstance(5, object)
@@ -624,34 +571,10 @@
         raises(TypeError, hasattr, x, 42)
         raises(UnicodeError, hasattr, x, u'\u5678')  # cannot encode attr name
 
-    def test_compile_leading_newlines(self):
-        src = """
-def fn(): pass
-"""
-        co = compile(src, 'mymod', 'exec')
-        firstlineno = co.co_firstlineno
-        assert firstlineno == 2
-
-    def test_compile_null_bytes(self):
-        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        src = "#abc\x00def\n"
-        raises(TypeError, compile, src, 'mymod', 'exec')
-        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+    def test_execfile_args(self):
         execfile(self.nullbytes) # works
-
-    def test_execfile_args(self):
         raises(TypeError, execfile, self.nonexistent, {}, ())
 
-    def test_compile_null_bytes_flag(self):
-        try:
-            from _ast import PyCF_ACCEPT_NULL_BYTES
-        except ImportError:
-            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               PyCF_ACCEPT_NULL_BYTES)
-        src = "#abc\x00def\n"
-        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
-
     def test_print_function(self):
         import __builtin__
         import sys
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -0,0 +1,77 @@
+class AppTestCompile:
+    def test_simple(self):
+        co = compile('1+2', '?', 'eval')
+        assert eval(co) == 3
+        co = compile(buffer('1+2'), '?', 'eval')
+        assert eval(co) == 3
+        exc = raises(TypeError, compile, chr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
+        assert str(exc.value) == "expected a readable buffer object"
+        compile("from __future__ import with_statement", "<test>", "exec")
+        raises(SyntaxError, compile, '-', '?', 'eval')
+        raises(ValueError, compile, '"\\xt"', '?', 'eval')
+        raises(ValueError, compile, '1+2', '?', 'maybenot')
+        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
+        raises(TypeError, compile, '1+2', 12, 34)
+
+    def test_error_message(self):
+        import re
+        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
+        exc = raises(SyntaxError, compile,
+            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        assert 'iso-8859-15' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+
+    def test_unicode(self):
+        try:
+            compile(u'-', '?', 'eval')
+        except SyntaxError as e:
+            assert e.lineno == 1
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        raises(SyntaxError, compile, code, "tmp", "exec")
+
+    def test_recompile_ast(self):
+        import _ast
+        # raise exception when node type doesn't match with compile mode
+        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
+        raises(TypeError, compile, co1, '<ast>', 'eval')
+        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
+        compile(co2, '<ast>', 'eval')
+
+    def test_leading_newlines(self):
+        src = """
+def fn(): pass
+"""
+        co = compile(src, 'mymod', 'exec')
+        firstlineno = co.co_firstlineno
+        assert firstlineno == 2
+
+    def test_null_bytes(self):
+        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        src = "#abc\x00def\n"
+        raises(TypeError, compile, src, 'mymod', 'exec')
+        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+
+    def test_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -20,6 +20,12 @@
         x = Static(1)
         assert isinstance(x, Static)
 
+        class C(Static):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_classmethod(self):
         class C(object):
             def f(cls, stuff):
@@ -41,8 +47,14 @@
         x = Classm(1)
         assert isinstance(x, Classm)
 
+        class C(Classm):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_property_simple(self):
-        
+
         class a(object):
             def _get(self): return 42
             def _set(self, value): raise AttributeError
@@ -98,7 +110,7 @@
             assert message.startswith('super(type, obj): obj must be an instance or subtype of type')
 
     def test_super_various(self):
-        
+
         class A(object):
             def meth(self, a):
                 return "A(%r)" % a
@@ -352,10 +364,10 @@
 
     def test_property_subclass_with_init(self):
         l = []
-        
+
         def x(self):
             l.append('x')
-        
+
         class P(property):
             def __init__(self):
                 property.__init__(self, x)
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -86,7 +86,9 @@
         return space.w_None
     return space.get(w_descr, w_obj)
 
-def do_what_I_mean(space):
+def do_what_I_mean(space, w_crash=None):
+    if not space.is_none(w_crash):
+        raise ValueError    # RPython-level, uncaught
     return space.wrap(42)
 
 
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -135,9 +135,19 @@
 class AppTestJitFeatures(object):
     spaceconfig = {"translation.jit": True}
 
+    def setup_class(cls):
+        cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
+
     def test_jit_backend_features(self):
         from __pypy__ import jit_backend_features
         supported_types = jit_backend_features
         assert isinstance(supported_types, list)
         for x in supported_types:
             assert x in ['floats', 'singlefloats', 'longlong']
+
+    def test_do_what_I_mean_error(self):
+        if not self.runappdirect:
+            skip("we don't wrap a random exception inside SystemError "
+                 "when untranslated, because it makes testing harder")
+        from __pypy__ import do_what_I_mean
+        raises(SystemError, do_what_I_mean, 1)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi

From pypy.commits at gmail.com  Mon Nov 14 06:06:01 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 Nov 2016 03:06:01 -0800 (PST)
Subject: [pypy-commit] pypy optinfo-into-bridges: need to set up the descrs
 more carefully here
Message-ID: <58299a99.49151c0a.9962c.c390@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: optinfo-into-bridges
Changeset: r88372:52ad37a21275
Date: 2016-11-14 12:05 +0100
http://bitbucket.org/pypy/pypy/changeset/52ad37a21275/

Log:	need to set up the descrs more carefully here

diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -79,6 +79,7 @@
 def test_compile_loop():
     cpu = FakeCPU()
     staticdata = FakeMetaInterpStaticData()
+    staticdata.all_descrs = LLtypeMixin.cpu.setup_descrs()
     staticdata.cpu = cpu
     staticdata.jitlog = jl.JitLogger(cpu)
     staticdata.jitlog.trace_id = 1

From pypy.commits at gmail.com  Mon Nov 14 08:28:14 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 14 Nov 2016 05:28:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: checkin of more changes to the cffi
 ssl module
Message-ID: <5829bbee.d4301c0a.163d9.f587@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88373:5cfd2ef1d12f
Date: 2016-11-14 14:27 +0100
http://bitbucket.org/pypy/pypy/changeset/5cfd2ef1d12f/

Log:	checkin of more changes to the cffi ssl module expose the raw
	address of the RawBuffer.buf field (as it is done for
	bytearrayobjects) allow only one cffi module to be built (added
	--only cli param)

diff too long, truncating to 2000 out of 2604 lines

diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -23,6 +23,9 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
+
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -54,12 +57,15 @@
 SIGNED_CERTFILE = data_file("keycert3.pem")
 SIGNED_CERTFILE2 = data_file("keycert4.pem")
 SIGNING_CA = data_file("pycacert.pem")
-
-SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem")
+# cert with all kinds of subject alt names
+ALLSANFILE = data_file("allsans.pem")
+
+REMOTE_HOST = "self-signed.pythontest.net"
+REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem")
 
 EMPTYCERT = data_file("nullcert.pem")
 BADCERT = data_file("badcert.pem")
-WRONGCERT = data_file("XXXnonexisting.pem")
+NONEXISTINGCERT = data_file("XXXnonexisting.pem")
 BADKEY = data_file("badkey.pem")
 NOKIACERT = data_file("nokia.pem")
 NULLBYTECERT = data_file("nullbytecert.pem")
@@ -142,8 +148,8 @@
     def test_str_for_enums(self):
         # Make sure that the PROTOCOL_* constants have enum-like string
         # reprs.
-        proto = ssl.PROTOCOL_SSLv23
-        self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_SSLv23')
+        proto = ssl.PROTOCOL_TLS
+        self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS')
         ctx = ssl.SSLContext(proto)
         self.assertIs(ctx.protocol, proto)
 
@@ -275,8 +281,29 @@
 
         self.assertEqual(p['subjectAltName'], san)
 
+    def test_parse_all_sans(self):
+        p = ssl._ssl._test_decode_cert(ALLSANFILE)
+        self.assertEqual(p['subjectAltName'],
+            (
+                ('DNS', 'allsans'),
+                ('othername', '<unsupported>'),
+                ('othername', '<unsupported>'),
+                ('email', 'user at example.org'),
+                ('DNS', 'www.example.org'),
+                ('DirName',
+                    ((('countryName', 'XY'),),
+                    (('localityName', 'Castle Anthrax'),),
+                    (('organizationName', 'Python Software Foundation'),),
+                    (('commonName', 'dirname example'),))),
+                ('URI', 'https://www.python.org/'),
+                ('IP Address', '127.0.0.1'),
+                ('IP Address', '0:0:0:0:0:0:0:1\n'),
+                ('Registered ID', '1.2.3.4.5')
+            )
+        )
+
     def test_DER_to_PEM(self):
-        with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f:
+        with open(CAFILE_CACERT, 'r') as f:
             pem = f.read()
         d1 = ssl.PEM_cert_to_DER_cert(pem)
         p2 = ssl.DER_cert_to_PEM_cert(d1)
@@ -311,8 +338,8 @@
         self.assertGreaterEqual(status, 0)
         self.assertLessEqual(status, 15)
         # Version string as returned by {Open,Libre}SSL, the format might change
-        if "LibreSSL" in s:
-            self.assertTrue(s.startswith("LibreSSL {:d}.{:d}".format(major, minor)),
+        if IS_LIBRESSL:
+            self.assertTrue(s.startswith("LibreSSL {:d}".format(major)),
                             (s, t, hex(n)))
         else:
             self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)),
@@ -366,17 +393,42 @@
                                     s.connect, (HOST, 8080))
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT)
+                ssl.wrap_socket(sock, certfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=CERTFILE, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def bad_cert_test(self, certfile):
+        """Check that trying to use the given client certificate fails"""
+        certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                   certfile)
+        sock = socket.socket()
+        self.addCleanup(sock.close)
+        with self.assertRaises(ssl.SSLError):
+            ssl.wrap_socket(sock,
+                            certfile=certfile,
+                            ssl_version=ssl.PROTOCOL_TLSv1)
+
+    def test_empty_cert(self):
+        """Wrapping with an empty cert file"""
+        self.bad_cert_test("nullcert.pem")
+
+    def test_malformed_cert(self):
+        """Wrapping with a badly formatted certificate (syntax error)"""
+        self.bad_cert_test("badcert.pem")
+
+    def test_malformed_key(self):
+        """Wrapping with a badly formatted key (syntax error)"""
+        self.bad_cert_test("badkey.pem")
+
     def test_match_hostname(self):
         def ok(cert, hostname):
             ssl.match_hostname(cert, hostname)
@@ -764,7 +816,8 @@
     def test_constructor(self):
         for protocol in PROTOCOLS:
             ssl.SSLContext(protocol)
-        self.assertRaises(TypeError, ssl.SSLContext)
+        ctx = ssl.SSLContext()
+        self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS)
         self.assertRaises(ValueError, ssl.SSLContext, -1)
         self.assertRaises(ValueError, ssl.SSLContext, 42)
 
@@ -785,17 +838,18 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
-            self.assertEqual(0, ctx.options)
+            # Ubuntu has OP_NO_SSLv3 forced on by default
+            self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3)
         else:
             with self.assertRaises(ValueError):
                 ctx.options = 0
@@ -842,7 +896,7 @@
         ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE)
         self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE)
         with self.assertRaises(OSError) as cm:
-            ctx.load_cert_chain(WRONGCERT)
+            ctx.load_cert_chain(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_cert_chain(BADCERT)
@@ -862,7 +916,7 @@
         # Mismatching key and cert
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"):
-            ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY)
+            ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY)
         # Password protected key and cert
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD)
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode())
@@ -927,7 +981,7 @@
         self.assertRaises(TypeError, ctx.load_verify_locations)
         self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None)
         with self.assertRaises(OSError) as cm:
-            ctx.load_verify_locations(WRONGCERT)
+            ctx.load_verify_locations(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_verify_locations(BADCERT)
@@ -1003,7 +1057,7 @@
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
         with self.assertRaises(FileNotFoundError) as cm:
-            ctx.load_dh_params(WRONGCERT)
+            ctx.load_dh_params(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(ssl.SSLError) as cm:
             ctx.load_dh_params(CERTFILE)
@@ -1080,7 +1134,7 @@
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 0, 'crl': 0, 'x509': 1})
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 1, 'crl': 0, 'x509': 2})
 
@@ -1090,8 +1144,8 @@
         # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.get_ca_certs(), [])
-        # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        # but CAFILE_CACERT is a CA cert
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.get_ca_certs(),
             [{'issuer': ((('organizationName', 'Root CA'),),
                          (('organizationalUnitName', 'http://www.cacert.org'),),
@@ -1107,7 +1161,7 @@
                           (('emailAddress', 'support at cacert.org'),)),
               'version': 3}])
 
-        with open(SVN_PYTHON_ORG_ROOT_CERT) as f:
+        with open(CAFILE_CACERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
         self.assertEqual(ctx.get_ca_certs(True), [der])
@@ -1128,6 +1182,7 @@
         self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH')
 
     @unittest.skipIf(sys.platform == "win32", "not-Windows specific")
+    @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars")
     def test_load_default_certs_env(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with support.EnvironmentVarGuard() as env:
@@ -1345,11 +1400,11 @@
 class NetworkedTests(unittest.TestCase):
 
     def test_connect(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
@@ -1358,27 +1413,27 @@
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED)
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                   s.connect, ("svn.python.org", 443))
+                                   s.connect, (REMOTE_HOST, 443))
             s.close()
 
             # this should succeed because we specify the root cert
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
 
     def test_connect_ex(self):
         # Issue #11326: check connect_ex() implementation
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                self.assertEqual(0, s.connect_ex(("svn.python.org", 443)))
+                self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443)))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
@@ -1386,14 +1441,14 @@
     def test_non_blocking_connect_ex(self):
         # Issue #11326: non-blocking connect_ex() should allow handshake
         # to proceed after the socket gets ready.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.setblocking(False)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
                 self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
                 # Wait for connect to finish
@@ -1415,58 +1470,62 @@
     def test_timeout_connect_ex(self):
         # Issue #12065: on a timeout, connect_ex() should return the original
         # errno (mimicking the behaviour of non-SSL sockets).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.settimeout(0.0000001)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 if rc == 0:
-                    self.skipTest("svn.python.org responded too quickly")
+                    self.skipTest("REMOTE_HOST responded too quickly")
                 self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK))
             finally:
                 s.close()
 
     def test_connect_ex_error(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                rc = s.connect_ex(("svn.python.org", 444))
+                rc = s.connect_ex((REMOTE_HOST, 444))
                 # Issue #19919: Windows machines or VMs hosted on Windows
                 # machines sometimes return EWOULDBLOCK.
-                self.assertIn(rc, (errno.ECONNREFUSED, errno.EWOULDBLOCK))
+                errors = (
+                    errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT,
+                    errno.EWOULDBLOCK,
+                )
+                self.assertIn(rc, errors)
             finally:
                 s.close()
 
     def test_connect_with_context(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             # Same as test_connect, but with a separately created context
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
             # Same with a server hostname
             s = ctx.wrap_socket(socket.socket(socket.AF_INET),
-                                server_hostname="svn.python.org")
-            s.connect(("svn.python.org", 443))
+                                server_hostname=REMOTE_HOST)
+            s.connect((REMOTE_HOST, 443))
             s.close()
             # This should fail because we have no verification certs
             ctx.verify_mode = ssl.CERT_REQUIRED
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                    s.connect, ("svn.python.org", 443))
+                                    s.connect, (REMOTE_HOST, 443))
             s.close()
             # This should succeed because we specify the root cert
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1479,12 +1538,12 @@
         # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must
         # contain both versions of each certificate (same content, different
         # filename) for this test to be portable across OpenSSL releases.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1495,7 +1554,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=BYTES_CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1503,15 +1562,15 @@
                 s.close()
 
     def test_connect_cadata(self):
-        with open(CAFILE_CACERT) as f:
+        with open(REMOTE_ROOT_CERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=pem)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1520,7 +1579,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=der)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1529,9 +1588,9 @@
         # Issue #5238: creating a file-like object with makefile() shouldn't
         # delay closing the underlying "real socket" (here tested with its
         # file descriptor, hence skipping the test under Windows).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
-            ss.connect(("svn.python.org", 443))
+            ss.connect((REMOTE_HOST, 443))
             fd = ss.fileno()
             f = ss.makefile()
             f.close()
@@ -1545,9 +1604,9 @@
             self.assertEqual(e.exception.errno, errno.EBADF)
 
     def test_non_blocking_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = socket.socket(socket.AF_INET)
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             s.setblocking(False)
             s = ssl.wrap_socket(s,
                                 cert_reqs=ssl.CERT_NONE,
@@ -1590,12 +1649,12 @@
                 if support.verbose:
                     sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))
 
-        _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT)
+        _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT)
         if support.IPV6_ENABLED:
             _test_get_server_certificate('ipv6.google.com', 443)
 
     def test_ciphers(self):
-        remote = ("svn.python.org", 443)
+        remote = (REMOTE_HOST, 443)
         with support.transient_internet(remote[0]):
             with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                  cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
@@ -1640,13 +1699,13 @@
 
     def test_get_ca_certs_capath(self):
         # capath certs are loaded on request
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             self.assertEqual(ctx.get_ca_certs(), [])
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1657,12 +1716,12 @@
     @needs_sni
     def test_context_setget(self):
         # Check that the context of a connected socket can be replaced.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = socket.socket(socket.AF_INET)
             with ctx1.wrap_socket(s) as ss:
-                ss.connect(("svn.python.org", 443))
+                ss.connect((REMOTE_HOST, 443))
                 self.assertIs(ss.context, ctx1)
                 self.assertIs(ss._sslobj.context, ctx1)
                 ss.context = ctx2
@@ -1683,13 +1742,8 @@
             try:
                 ret = func(*args)
             except ssl.SSLError as e:
-                # Note that we get a spurious -1/SSL_ERROR_SYSCALL for
-                # non-blocking IO. The SSL_shutdown manpage hints at this.
-                # It *should* be safe to just ignore SYS_ERROR_SYSCALL because
-                # with a Memory BIO there's no syscalls (for IO at least).
                 if e.errno not in (ssl.SSL_ERROR_WANT_READ,
-                                   ssl.SSL_ERROR_WANT_WRITE,
-                                   ssl.SSL_ERROR_SYSCALL):
+                                   ssl.SSL_ERROR_WANT_WRITE):
                     raise
                 errno = e.errno
             # Get any data from the outgoing BIO irrespective of any error, and
@@ -1712,36 +1766,42 @@
         return ret
 
     def test_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             ctx.check_hostname = True
-            sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org')
+            sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            self.assertIsNotNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            self.assertIsNotNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
-            self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            try:
+                self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            except ssl.SSLSyscallError:
+                # self-signed.pythontest.net probably shuts down the TCP
+                # connection without sending a secure shutdown message, and
+                # this is reported as SSL_ERROR_SYSCALL
+                pass
             self.assertRaises(ssl.SSLError, sslobj.write, b'foo')
             sock.close()
 
     def test_read_write_data(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -2084,36 +2144,6 @@
             self.active = False
             self.server.close()
 
-    def bad_cert_test(certfile):
-        """
-        Launch a server with CERT_REQUIRED, and check that trying to
-        connect to it with the given client certificate fails.
-        """
-        server = ThreadedEchoServer(CERTFILE,
-                                    certreqs=ssl.CERT_REQUIRED,
-                                    cacerts=CERTFILE, chatty=False,
-                                    connectionchatty=False)
-        with server:
-            try:
-                with socket.socket() as sock:
-                    s = ssl.wrap_socket(sock,
-                                        certfile=certfile,
-                                        ssl_version=ssl.PROTOCOL_TLSv1)
-                    s.connect((HOST, server.port))
-            except ssl.SSLError as x:
-                if support.verbose:
-                    sys.stdout.write("\nSSLError is %s\n" % x.args[1])
-            except OSError as x:
-                if support.verbose:
-                    sys.stdout.write("\nOSError is %s\n" % x.args[1])
-            except OSError as x:
-                if x.errno != errno.ENOENT:
-                    raise
-                if support.verbose:
-                    sys.stdout.write("\OSError is %s\n" % str(x))
-            else:
-                raise AssertionError("Use of invalid cert should have failed!")
-
     def server_params_test(client_context, server_context, indata=b"FOO\n",
                            chatty=True, connectionchatty=False, sni_name=None):
         """
@@ -2354,22 +2384,38 @@
                                                 "check_hostname requires server_hostname"):
                         context.wrap_socket(s)
 
-        def test_empty_cert(self):
-            """Connecting with an empty cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                      "nullcert.pem"))
-        def test_malformed_cert(self):
-            """Connecting with a badly formatted certificate (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badcert.pem"))
-        def test_nonexisting_cert(self):
-            """Connecting with a non-existing cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "wrongcert.pem"))
-        def test_malformed_key(self):
-            """Connecting with a badly formatted key (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badkey.pem"))
+        def test_wrong_cert(self):
+            """Connecting when the server rejects the client's certificate
+
+            Launch a server with CERT_REQUIRED, and check that trying to
+            connect to it with a wrong client certificate fails.
+            """
+            certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                       "wrongcert.pem")
+            server = ThreadedEchoServer(CERTFILE,
+                                        certreqs=ssl.CERT_REQUIRED,
+                                        cacerts=CERTFILE, chatty=False,
+                                        connectionchatty=False)
+            with server, \
+                    socket.socket() as sock, \
+                    ssl.wrap_socket(sock,
+                                        certfile=certfile,
+                                        ssl_version=ssl.PROTOCOL_TLSv1) as s:
+                try:
+                    # Expect either an SSL error about the server rejecting
+                    # the connection, or a low-level connection reset (which
+                    # sometimes happens on Windows)
+                    s.connect((HOST, server.port))
+                except ssl.SSLError as e:
+                    if support.verbose:
+                        sys.stdout.write("\nSSLError is %r\n" % e)
+                except OSError as e:
+                    if e.errno != errno.ECONNRESET:
+                        raise
+                    if support.verbose:
+                        sys.stdout.write("\nsocket.error is %r\n" % e)
+                else:
+                    self.fail("Use of invalid cert should have failed!")
 
         def test_rude_shutdown(self):
             """A brutal shutdown of an SSL server should raise an OSError
@@ -2615,7 +2661,7 @@
                     s.close()
 
         def test_socketserver(self):
-            """Using a SocketServer to create and manage SSL connections."""
+            """Using socketserver to create and manage SSL connections."""
             server = make_https_server(self, certfile=CERTFILE)
             # try to connect
             if support.verbose:
@@ -2642,8 +2688,6 @@
 
         def test_asyncore_server(self):
             """Check the example asyncore integration."""
-            indata = "TEST MESSAGE of mixed case\n"
-
             if support.verbose:
                 sys.stdout.write("\n")
 
@@ -2775,6 +2819,13 @@
                         # consume data
                         s.read()
 
+                # read(-1, buffer) is supported, even though read(-1) is not
+                data = b"data"
+                s.send(data)
+                buffer = bytearray(len(data))
+                self.assertEqual(s.read(-1, buffer), len(data))
+                self.assertEqual(buffer, data)
+
                 # Make sure sendmsg et al are disallowed to avoid
                 # inadvertent disclosure of data and/or corruption
                 # of the encrypted data stream
@@ -2784,8 +2835,32 @@
                                   s.recvmsg_into, bytearray(100))
 
                 s.write(b"over\n")
+
+                self.assertRaises(ValueError, s.recv, -1)
+                self.assertRaises(ValueError, s.read, -1)
+
                 s.close()
 
+        def test_recv_zero(self):
+            server = ThreadedEchoServer(CERTFILE)
+            server.__enter__()
+            self.addCleanup(server.__exit__, None, None)
+            s = socket.create_connection((HOST, server.port))
+            self.addCleanup(s.close)
+            s = ssl.wrap_socket(s, suppress_ragged_eofs=False)
+            self.addCleanup(s.close)
+
+            # recv/read(0) should return no data
+            s.send(b"data")
+            self.assertEqual(s.recv(0), b"")
+            self.assertEqual(s.read(0), b"")
+            self.assertEqual(s.read(), b"data")
+
+            # Should not block if the other end sends no data
+            s.setblocking(False)
+            self.assertEqual(s.recv(0), b"")
+            self.assertEqual(s.recv_into(bytearray()), 0)
+
         def test_nonblocking_send(self):
             server = ThreadedEchoServer(CERTFILE,
                                         certreqs=ssl.CERT_NONE,
@@ -2944,7 +3019,7 @@
                 with context.wrap_socket(socket.socket()) as s:
                     self.assertIs(s.version(), None)
                     s.connect((HOST, server.port))
-                    self.assertEqual(s.version(), "TLSv1")
+                    self.assertEqual(s.version(), 'TLSv1')
                 self.assertIs(s.version(), None)
 
         @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL")
@@ -3086,24 +3161,36 @@
                 (['http/3.0', 'http/4.0'], None)
             ]
             for client_protocols, expected in protocol_tests:
-                server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+                server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
                 server_context.load_cert_chain(CERTFILE)
                 server_context.set_alpn_protocols(server_protocols)
-                client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+                client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+
+                try:
+                    stats = server_params_test(client_context,
+                                               server_context,
+                                               chatty=True,
+                                               connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                        "was expecting %s, but got %%s from the %%s" \
+                            % (str(server_protocols), str(client_protocols),
+                                str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
@@ -3251,13 +3338,23 @@
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            client_context.set_ciphers("RC4")
-            server_context.set_ciphers("AES:RC4")
+            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
+                client_context.set_ciphers("AES128:AES256")
+                server_context.set_ciphers("AES256")
+                alg1 = "AES256"
+                alg2 = "AES-256"
+            else:
+                client_context.set_ciphers("AES:3DES")
+                server_context.set_ciphers("3DES")
+                alg1 = "3DES"
+                alg2 = "DES-CBC3"
+
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                self.assertIn("RC4", name.split("-"))
+                if not alg1 in name.split("-") and alg2 not in name:
+                    self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -3325,7 +3422,7 @@
             pass
 
     for filename in [
-        CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE,
+        CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE,
         ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY,
         SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA,
         BADCERT, BADKEY, EMPTYCERT]:
diff --git a/lib_pypy/openssl/_cffi_src/build_openssl.py b/lib_pypy/openssl/_cffi_src/build_openssl.py
--- a/lib_pypy/openssl/_cffi_src/build_openssl.py
+++ b/lib_pypy/openssl/_cffi_src/build_openssl.py
@@ -73,6 +73,7 @@
         "rand",
         "rsa",
         "ssl",
+        "tls1",
         "x509",
         "x509name",
         "x509v3",
diff --git a/lib_pypy/openssl/_cffi_src/openssl/bio.py b/lib_pypy/openssl/_cffi_src/openssl/bio.py
--- a/lib_pypy/openssl/_cffi_src/openssl/bio.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/bio.py
@@ -130,6 +130,8 @@
 long BIO_set_buffer_size(BIO *, long);
 long BIO_set_buffer_read_data(BIO *, void *, long);
 long BIO_set_nbio(BIO *, long);
+void BIO_set_retry_read(BIO *);
+void BIO_clear_retry_flags(BIO *);
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -133,15 +133,16 @@
 typedef ... SSL_METHOD;
 typedef ... SSL_CTX;
 
+
 typedef ... SSL_SESSION;
+typedef ... SSL;
 
-typedef ... SSL;
+typedef ... Cryptography_STACK_OF_SSL_CIPHER;
 
 static const long TLSEXT_NAMETYPE_host_name;
 
+typedef ... COMP_METHOD;
 typedef ... SSL_CIPHER;
-typedef ... Cryptography_STACK_OF_SSL_CIPHER;
-typedef ... COMP_METHOD;
 """
 
 FUNCTIONS = """
@@ -193,6 +194,7 @@
 int SSL_renegotiate_pending(SSL *);
 const char *SSL_get_cipher_list(const SSL *, int);
 Cryptography_STACK_OF_SSL_CIPHER *SSL_get_ciphers(const SSL *);
+Cryptography_STACK_OF_SSL_CIPHER * Cryptography_get_ssl_session_ciphers(const SSL_SESSION *);
 
 /*  context */
 void SSL_CTX_free(SSL_CTX *);
@@ -242,6 +244,8 @@
 Cryptography_STACK_OF_X509_NAME *SSL_load_client_CA_file(const char *);
 
 const char *SSL_get_servername(const SSL *, const int);
+//long SSL_CTX_callback_ctrl(SSL_CTX *, int, void (*)(void));
+//long SSL_CTX_ctrl(SSL_CTX *, int, long, void *);
 """
 
 MACROS = """
@@ -703,4 +707,8 @@
 static const long TLS_ST_BEFORE = 0;
 static const long TLS_ST_OK = 0;
 #endif
+
+Cryptography_STACK_OF_SSL_CIPHER * Cryptography_get_ssl_session_ciphers(const SSL_SESSION *s) {
+    return s->ciphers;
+}
 """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509.py b/lib_pypy/openssl/_cffi_src/openssl/x509.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509.py
@@ -18,6 +18,7 @@
 typedef STACK_OF(X509_CRL) Cryptography_STACK_OF_X509_CRL;
 typedef STACK_OF(X509_REVOKED) Cryptography_STACK_OF_X509_REVOKED;
 typedef STACK_OF(ACCESS_DESCRIPTION) Cryptography_AUTHORITY_INFO_ACCESS;
+typedef STACK_OF(X509_OBJECT) Cryptography_STACK_OF_X509_OBJECT;
 """
 
 TYPES = """
@@ -340,6 +341,9 @@
                              const X509_ALGOR **);
 
 void AUTHORITY_INFO_ACCESS_free(Cryptography_AUTHORITY_INFO_ACCESS*);
+
+#define X509_R_CERT_ALREADY_IN_HASH_TABLE ...
+#define ASN1_R_HEADER_TOO_LONG ...
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
@@ -15,6 +15,7 @@
  * Note that the result is an opaque type.
  */
 typedef STACK_OF(ASN1_OBJECT) Cryptography_STACK_OF_ASN1_OBJECT;
+typedef STACK_OF(X509_OBJECT) Cryptography_STACK_OF_X509_OBJECT;
 """
 
 TYPES = """
@@ -22,10 +23,17 @@
 static const long Cryptography_HAS_102_VERIFICATION_PARAMS;
 static const long Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST;
 static const long Cryptography_HAS_X509_V_FLAG_PARTIAL_CHAIN;
+static const long Cryptography_X509_LU_X509;
+static const long Cryptography_X509_LU_CLR;
 
 typedef ... Cryptography_STACK_OF_ASN1_OBJECT;
+typedef ... Cryptography_STACK_OF_X509_OBJECT;
 
-typedef ... X509_STORE;
+typedef ... X509_OBJECT;
+typedef struct {
+    Cryptography_STACK_OF_X509_OBJECT * objs;
+    ...;
+} X509_STORE;
 typedef ... X509_VERIFY_PARAM;
 typedef ... X509_STORE_CTX;
 
@@ -119,6 +127,7 @@
 static const long X509_V_FLAG_SUITEB_192_LOS;
 static const long X509_V_FLAG_SUITEB_128_LOS;
 static const long X509_V_FLAG_PARTIAL_CHAIN;
+
 """
 
 FUNCTIONS = """
@@ -172,6 +181,9 @@
                                     Cryptography_STACK_OF_ASN1_OBJECT *);
 void X509_VERIFY_PARAM_set_depth(X509_VERIFY_PARAM *, int);
 int X509_VERIFY_PARAM_get_depth(const X509_VERIFY_PARAM *);
+
+int Cryptography_X509_OBJECT_get_type(const X509_OBJECT *);
+X509 * Cryptography_X509_OBJECT_data_x509(X509_OBJECT*);
 """
 
 MACROS = """
@@ -192,6 +204,11 @@
 int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *, const unsigned char *,
                               size_t);
 int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *, const char *);
+
+/* STACK_OF(X509_OBJECT) */
+int sk_X509_OBJECT_num(Cryptography_STACK_OF_X509_OBJECT *);
+X509_OBJECT *sk_X509_OBJECT_value(Cryptography_STACK_OF_X509_OBJECT *, int);
+
 """
 
 CUSTOMIZATIONS = """
@@ -254,4 +271,15 @@
 X509_VERIFY_PARAM *_X509_STORE_get0_param(X509_STORE *store) {
     return store->param;
 }
+
+int Cryptography_X509_OBJECT_get_type(const X509_OBJECT * x) {
+    return x->type;
+}
+
+X509 * Cryptography_X509_OBJECT_data_x509(X509_OBJECT * x) {
+    return x->data.x509;
+}
+
+static const long Cryptography_X509_LU_X509 = X509_LU_X509;
+static const long Cryptography_X509_LU_CLR = X509_LU_CRL;
 """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509v3.py b/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
@@ -185,7 +185,7 @@
 void *X509V3_EXT_d2i(X509_EXTENSION *);
 /* X509_get_ext_d2i is already defined, there might be a better solution
    to expose it to the lib object?  */
-void * _X509_get_ext_d2i(const X509 *, int, int *, int *);
+void * _X509_get_ext_d2i(X509 *, int, int *, int *);
 /* X509 is private, there is no way to access the field crldp other than
    adding it to the typedef or expose a function like this: */
 Cryptography_STACK_OF_DIST_POINT * _X509_get_crldp(const X509 *);
@@ -305,7 +305,7 @@
 """
 
 CUSTOMIZATIONS = """
-void * _X509_get_ext_d2i(const X509 * x, int nid, int * a, int * b) {
+void * _X509_get_ext_d2i(X509 * x, int nid, int * a, int * b) {
     return X509_get_ext_d2i(x, nid, a, b);
 }
 #if OPENSSL_VERSION_NUMBER >= 0x10001000L
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -1,16 +1,22 @@
 import sys
-import errno
 import time
 import _thread
+import socket
 import weakref
 from _openssl import ffi
 from _openssl import lib
-from openssl._stdssl.certificate import _test_decode_cert
-from openssl._stdssl.utility import _str_with_len
-from openssl._stdssl.error import (ssl_error, ssl_lib_error,
+from openssl._stdssl.certificate import (_test_decode_cert,
+    _decode_certificate, _certificate_to_der)
+from openssl._stdssl.utility import _str_with_len, _bytes_with_len, _str_to_ffi_buffer
+from openssl._stdssl.error import (ssl_error, ssl_lib_error, ssl_socket_error,
         SSLError, SSLZeroReturnError, SSLWantReadError,
         SSLWantWriteError, SSLSyscallError,
         SSLEOFError)
+from openssl._stdssl.error import (SSL_ERROR_NONE,
+        SSL_ERROR_SSL, SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE,
+        SSL_ERROR_WANT_X509_LOOKUP, SSL_ERROR_SYSCALL,
+        SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_CONNECT,
+        SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE)
 
 
 OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
@@ -121,22 +127,74 @@
 else:
     Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
 
-def _ssl_select(sock, write, timeout):
-    pass
-    raise NotImplementedError
+from select import poll, POLLIN, POLLOUT, select
+
+HAVE_POLL = True
+
+def _ssl_select(sock, writing, timeout):
+    if HAVE_POLL:
+        p = poll()
+
+    # Nothing to do unless we're in timeout mode (not non-blocking)
+    if sock is None or timeout == 0:
+        return SOCKET_IS_NONBLOCKING
+    elif timeout < 0:
+        t = sock.gettimeout() or 0
+        if t > 0:
+            return SOCKET_HAS_TIMED_OUT
+        else:
+            return SOCKET_IS_BLOCKING
+
+    # Guard against closed socket
+    if sock.fileno() < 0:
+        return SOCKET_HAS_BEEN_CLOSED
+
+    # Prefer poll, if available, since you can poll() any fd
+    # which can't be done with select().
+    if HAVE_POLL:
+        p.register(sock.fileno(), POLLOUT | POLLIN)
+
+        #PySSL_BEGIN_ALLOW_THREADS
+        rc = len(p.poll(timeout * 1000.0))
+        #PySSL_END_ALLOW_THREADS
+    else:
+        fd = sock.fileno()
+        #if (!_PyIsSelectable_fd(s->sock_fd))
+        #    return SOCKET_TOO_LARGE_FOR_SELECT;
+        if writing:
+            rr, wr, xr = select([],[fd],[], timeout)
+        else:
+            rr, wr, xr = select([fd],[],[], timeout)
+        rc = len(rr) + len(wr)
+    if rc != 0:
+        return SOCKET_OPERATION_OK
+    return SOCKET_HAS_TIMED_OUT
+
+SOCKET_IS_NONBLOCKING = 0
+SOCKET_IS_BLOCKING = 1
+SOCKET_HAS_TIMED_OUT = 2
+SOCKET_HAS_BEEN_CLOSED = 3
+SOCKET_TOO_LARGE_FOR_SELECT = 4
+SOCKET_OPERATION_OK = 5
+
+def _buffer_new(length):
+    return ffi.new("char[%d]"%length)
 
 class _SSLSocket(object):
 
     @staticmethod
-    def _new__ssl_socket(sslctx, sock, socket_type, hostname, inbio, outbio):
-        self = _SSLSocket(sslctx, sock, socket_type, hostname)
+    def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, inbio, outbio):
+        self = _SSLSocket(sslctx)
         ctx = sslctx.ctx
 
+        if server_hostname:
+            self.server_hostname = server_hostname.decode('idna', 'strict')
+
         lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = lib.SSL_new(ctx)
 
-        # TODO _server_name_callback self.SSL_set_app_data(self.ssl, self);
+        lib.SSL_set_app_data(ssl, b"")
         if sock:
             lib.SSL_set_fd(ssl, sock.fileno())
         else:
@@ -157,11 +215,11 @@
             name = _str_to_ffi_buffer(self.server_hostname)
             lib.SSL_set_tlsext_host_name(ssl, name)
 
-        timeout = sock.gettimeout() or -1
 
         # If the socket is in non-blocking mode or timeout mode, set the BIO
         # to non-blocking mode (blocking is the default)
         #
+        timeout = sock.gettimeout() or 0
         if sock and timeout >= 0:
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), 1)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), 1)
@@ -171,38 +229,43 @@
             lib.SSL_set_connect_state(ssl)
         else:
             lib.SSL_set_accept_state(ssl)
+        self.socket_type = socket_type
         #PySSL_END_ALLOW_THREADS
 
         if sock:
-            self.Socket = weakref.ref(sock)
+            self.socket = weakref.ref(sock)
 
         return self
 
-
-    def __init__(self, sslctx, sock, sockettype, hostname):
-        self.sock = sock
+    def __init__(self, sslctx):
+        self.ctx = sslctx
+        self.peer_cert = ffi.NULL
         self.ssl = ffi.NULL
-        self.sockettype = sockettype
-        self.ctx = sslctx
         self.shutdown_seen_zero = 0
         self.handshake_done = 0
         self.owner = None
-        if hostname:
-            self.server_hostname = hostname.decode('idna', 'strict')
-        else:
-            self.server_hostname = None
+        self.server_hostname = None
+        self.socket = None
+
+    @property
+    def context(self):
+        return self.ctx
+
+    @context.setter
+    def context(self, value):
+        self.ctx = value
 
     def do_handshake(self):
-
-        sock = self.sock
+        sock = self.get_socket_or_None()
         if sock is None:
-            _setSSLError("Underlying socket connection gone", lib.SSL_ERROR_NO_SOCKET)
-
+            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
         ssl = self.ssl
-
-        nonblocking = timeout >= 0
-        lib.BIO_set_nbio(lib.SSL_getrbio(ssl), nonblocking)
-        lib.BIO_set_nbio(lib.SSL_getwbio(ssl), nonblocking)
+        timeout = 0
+        if sock:
+            timeout = sock.gettimeout() or 0
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
         has_timeout = timeout > 0
         has_timeout = (timeout > 0);
@@ -210,7 +273,6 @@
         if has_timeout:
             # REVIEW, cpython uses a monotonic clock here
             deadline = time.time() + timeout;
-
         # Actually negotiate SSL connection
         # XXX If SSL_do_handshake() returns 0, it's also a failure.
         while True:
@@ -226,9 +288,9 @@
                 # REIVIEW monotonic clock?
                 timeout = deadline - time.time()
 
-            if err == lib.SSL_ERROR_WANT_READ:
+            if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
-            elif err == lib.SSL_ERROR_WANT_WRITE:
+            elif err == SSL_ERROR_WANT_WRITE:
                 sockstate = _ssl_select(sock, 1, timeout)
             else:
                 sockstate = SOCKET_OPERATION_OK
@@ -243,10 +305,10 @@
                 break
             if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
                 break
-        #if (ret < 1:
-        #    return PySSL_SetError(self, ret, __FILE__, __LINE__);
+        if ret < 1:
+            raise ssl_lib_error()
 
-        if self.peer_cert:
+        if self.peer_cert != ffi.NULL:
             lib.X509_free(self.peer_cert)
         #PySSL_BEGIN_ALLOW_THREADS
         self.peer_cert = lib.SSL_get_peer_certificate(ssl)
@@ -254,10 +316,329 @@
         self.handshake_done = 1
         return None
 
+    def peer_certificate(self, binary_mode):
+        if not self.handshake_done:
+            raise ValueError("handshake not done yet")
+        if self.peer_cert == ffi.NULL:
+            return None
 
+        if binary_mode:
+            # return cert in DER-encoded format
+            return _certificate_to_der(self.peer_cert)
+        else:
+            verification = lib.SSL_CTX_get_verify_mode(lib.SSL_get_SSL_CTX(self.ssl))
+            if (verification & lib.SSL_VERIFY_PEER) == 0:
+                return {}
+            else:
+                return _decode_certificate(self.peer_cert)
+
+    def write(self, bytestring):
+        deadline = 0
+        b = _str_to_ffi_buffer(bytestring)
+        sock = self.get_socket_or_None()
+        ssl = self.ssl
+        if sock:
+            timeout = sock.gettimeout() or 0
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+
+        timeout = sock.gettimeout() or 0
+        has_timeout = timeout > 0
+        if has_timeout:
+            # TODO monotonic clock?
+            deadline = time.time() + timeout
+
+        sockstate = _ssl_select(sock, 1, timeout)
+        if sockstate == SOCKET_HAS_TIMED_OUT:
+            raise socket.TimeoutError("The write operation timed out")
+        elif sockstate == SOCKET_HAS_BEEN_CLOSED:
+            raise ssl_error("Underlying socket has been closed.")
+        elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
+            raise ssl_error("Underlying socket too large for select().")
+
+        while True:
+            #PySSL_START_ALLOW_THREADS
+            length = lib.SSL_write(self.ssl, b, len(b))
+            err = lib.SSL_get_error(self.ssl, length)
+            #PySSL_END_ALLOW_THREADS
+
+            # TODO if (PyErr_CheckSignals())
+            # TODO     goto error;
+
+            if has_timeout:
+                # TODO monotonic clock
+                timeout = deadline - time.time()
+
+            if err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            else:
+                sockstate = SOCKET_OPERATION_OK
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                raise socket.TimeoutError("The write operation timed out")
+            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
+                raise ssl_error("Underlying socket has been closed.")
+            elif sockstate == SOCKET_IS_NONBLOCKING:
+                break
+            if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
+                break
+
+        if length > 0:
+            return length
+        else:
+            raise ssl_lib_error()
+            # return PySSL_SetError(self, len, __FILE__, __LINE__);
+
+    def read(self, length, buffer_into=None):
+        sock = self.get_socket_or_None()
+        ssl = self.ssl
+
+        if sock is None:
+            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+
+        if not buffer_into:
+            dest = _buffer_new(length)
+            mem = dest
+        else:
+            import pdb; pdb.set_trace()
+            mem = ffi.from_buffer(buffer_into)
+            if length <= 0 or length > len(buffer_into):
+                if len(buffer_into) != length:
+                    raise OverflowError("maximum length can't fit in a C 'int'")
+
+        if sock:
+            timeout = sock.gettimeout() or 0
+            nonblocking = timeout >= 0
+            lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+            lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+
+        deadline = 0
+        timeout = sock.gettimeout() or 0
+        has_timeout = timeout > 0
+        if has_timeout:
+            # TODO monotonic clock?
+            deadline = time.time() + timeout
+
+        shutdown = False
+        while True:
+            #PySSL_BEGIN_ALLOW_THREADS
+            count = lib.SSL_read(self.ssl, mem, length);
+            err = lib.SSL_get_error(self.ssl, count);
+            #PySSL_END_ALLOW_THREADS
+
+            # TODO
+            #if (PyErr_CheckSignals())
+            #    goto error;
+
+            if has_timeout:
+                timeout = deadline - time.time() # TODO ? _PyTime_GetMonotonicClock();
+
+            if err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            elif err == SSL_ERROR_ZERO_RETURN and \
+                 lib.SSL_get_shutdown(self.ssl) == lib.SSL_RECEIVED_SHUTDOWN:
+                shutdown = True
+                break;
+            else:
+                sockstate = SOCKET_OPERATION_OK
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                raise socket.TimeoutError("The read operation timed out")
+            elif sockstate == SOCKET_IS_NONBLOCKING:
+                break
+            if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
+                break
+
+        if count <= 0:
+            raise ssl_socket_error(self, err)
+
+        if not buffer_into:
+            return _bytes_with_len(dest, count)
+
+        return count
+
+    def selected_alpn_protocol(self):
+        out = ffi.new("const unsigned char **")
+        outlen = ffi.new("unsigned int*")
+
+        lib.SSL_get0_alpn_selected(self.ssl, out, outlen);
+        if out == ffi.NULL:
+            return None
+        return _str_with_len(ffi.cast("char*",out[0]), outlen[0]);
+
+    def shared_ciphers(self):
+        sess = lib.SSL_get_session(self.ssl)
+
+        ciphers = lib.Cryptography_get_ssl_session_ciphers(sess)
+        if sess is None or ciphers == ffi.NULL:
+            return None
+        res = []
+        count = lib.sk_SSL_CIPHER_num(ciphers)
+        for i in range(count):
+            tup = cipher_to_tuple(lib.sk_SSL_CIPHER_value(ciphers, i))
+            if not tup:
+                return None
+            res.append(tup)
+        return res
+
+    def cipher(self):
+        if self.ssl == ffi.NULL:
+            return None
+        current = lib.SSL_get_current_cipher(self.ssl)
+        if current == ffi.NULL:
+            return None
+        return cipher_to_tuple(current)
+
+    def compression(self):
+        if not lib.Cryptography_HAS_COMPRESSION or self.ssl == ffi.NULL:
+            return None
+
+        comp_method = lib.SSL_get_current_compression(self.ssl);
+        if comp_method == ffi.NULL: # or comp_method.type == lib.NID_undef:
+            return None
+        short_name = lib.SSL_COMP_get_name(comp_method)
+        if short_name == ffi.NULL:
+            return None
+        return _fs_decode(_str_from_buf(short_name))
+
+    def version(self):
+        if self.ssl == ffi.NULL:
+            return None
+        version = _str_from_buf(lib.SSL_get_version(self.ssl))
+        if version == "unknown":
+            return None
+        return version
+
+    def get_socket_or_None(self):
+        if self.socket is None:
+            return None
+        return self.socket()
+
+    def shutdown(self):
+        sock = self.get_socket_or_None()
+        nonblocking = False
+        ssl = self.ssl
+
+        if sock is not None:
+            # Guard against closed socket
+            if sock.fileno() < 0:
+                raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+
+            timeout = sock.gettimeout() or 0
+            nonblocking = timeout >= 0
+            if sock and timeout >= 0:
+                lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
+                lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
+        else:
+            timeout = 0
+
+        has_timeout = (timeout > 0);
+        if has_timeout:
+            # TODO monotonic clock
+            deadline = time.time() + timeout;
+
+        zeros = 0
+
+        while True:
+            # TODO PySSL_BEGIN_ALLOW_THREADS
+            # Disable read-ahead so that unwrap can work correctly.
+            # Otherwise OpenSSL might read in too much data,
+            # eating clear text data that happens to be
+            # transmitted after the SSL shutdown.
+            # Should be safe to call repeatedly every time this
+            # function is used and the shutdown_seen_zero != 0
+            # condition is met.
+            #
+            if self.shutdown_seen_zero:
+                lib.SSL_set_read_ahead(self.ssl, 0)
+            err = lib.SSL_shutdown(self.ssl)
+            # TODO PySSL_END_ALLOW_THREADS
+
+            # If err == 1, a secure shutdown with SSL_shutdown() is complete
+            if err > 0:
+                break
+            if err == 0:
+                # Don't loop endlessly; instead preserve legacy
+                #   behaviour of trying SSL_shutdown() only twice.
+                #   This looks necessary for OpenSSL < 0.9.8m
+                zeros += 1
+                if zeros > 1:
+                    break
+                # Shutdown was sent, now try receiving
+                self.shutdown_seen_zero = 1
+                continue
+
+            if has_timeout:
+                # TODO monotonic clock
+                timeout = deadline - time.time() #_PyTime_GetMonotonicClock();
+
+            # Possibly retry shutdown until timeout or failure
+            ssl_err = lib.SSL_get_error(self.ssl, err)
+            if ssl_err == SSL_ERROR_WANT_READ:
+                sockstate = _ssl_select(sock, 0, timeout)
+            elif ssl_err == SSL_ERROR_WANT_WRITE:
+                sockstate = _ssl_select(sock, 1, timeout)
+            else:
+                break
+
+            if sockstate == SOCKET_HAS_TIMED_OUT:
+                if ssl_err == SSL_ERROR_WANT_READ:
+                    raise socket.TimeoutError("The read operation timed out")
+                else:
+                    raise socket.TimeoutError("The write operation timed out")
+            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
+                raise ssl_error("Underlying socket too large for select().")
+            elif sockstate != SOCKET_OPERATION_OK:
+                # Retain the SSL error code
+                break;
+
+        if err < 0:
+            raise ssl_socket_error(self, err)
+        if sock:
+            return sock
+        else:
+            return None
+
+
+def _fs_decode(name):
+    # TODO return PyUnicode_DecodeFSDefault(short_name);
+    return name.decode('utf-8')
+
+
+def cipher_to_tuple(cipher):
+    ccipher_name = lib.SSL_CIPHER_get_name(cipher)
+    if ccipher_name == ffi.NULL:
+        cipher_name = None
+    else:
+        cipher_name = _str_from_buf(ccipher_name)
+
+    ccipher_protocol = lib.SSL_CIPHER_get_version(cipher)
+    if ccipher_protocol == ffi.NULL:
+        cipher_protocol = None
+    else:
+        cipher_protocol = _str_from_buf(ccipher_protocol)
+
+    bits = lib.SSL_CIPHER_get_bits(cipher, ffi.NULL)
+    return (cipher_name, cipher_protocol, bits)
+
+
+
+SSL_CTX_STATS_NAMES = """
+    number connect connect_good connect_renegotiate accept accept_good
+    accept_renegotiate hits misses timeouts cache_full""".split()
+SSL_CTX_STATS = []
+for name in SSL_CTX_STATS_NAMES:
+    attr = 'SSL_CTX_sess_'+name
+    assert hasattr(lib, attr)
+    SSL_CTX_STATS.append((name, getattr(lib, attr)))
 
 class _SSLContext(object):
-    __slots__ = ('ctx', 'check_hostname')
+    __slots__ = ('ctx', '_check_hostname', 'servername_callback')
 
     def __new__(cls, protocol):
         self = object.__new__(cls)
@@ -281,7 +662,7 @@
         if self.ctx == ffi.NULL: 
             raise ssl_error("failed to allocate SSL context")
 
-        self.check_hostname = False
+        self._check_hostname = False
         # TODO self.register_finalizer(space)
 
         # Defaults
@@ -304,8 +685,7 @@
             else:
                 key = lib.EC_KEY_new_by_curve_name(lib.NID_X9_62_prime256v1)
                 if not key:
-                    # TODO copy from ropenssl?
-                    raise _ssl_seterror(None, 0)
+                    raise ssl_lib_error()
                 try:
                     lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
                 finally:
@@ -384,6 +764,18 @@
             if not lib.X509_VERIFY_PARAM_set_flags(param, set):
                 raise ssl_error(None, 0)
 
+    @property
+    def check_hostname(self):
+        return self._check_hostname
+
+    @check_hostname.setter
+    def check_hostname(self, value):
+        check_hostname = bool(value)
+        if check_hostname and lib.SSL_CTX_get_verify_mode(self.ctx) == lib.SSL_VERIFY_NONE:
+            raise ValueError("check_hostname needs a SSL context with either "
+                             "CERT_OPTIONAL or CERT_REQUIRED")
+        self._check_hostname = check_hostname
+
     def set_ciphers(self, cipherlist):
         cipherlistbuf = _str_to_ffi_buffer(cipherlist)
         ret = lib.SSL_CTX_set_cipher_list(self.ctx, cipherlistbuf)
@@ -416,6 +808,7 @@
             lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.new_handle(pw_info))
 
         try:
+            ffi.errno = 0
             certfilebuf = _str_to_ffi_buffer(certfile)
             ret = lib.SSL_CTX_use_certificate_chain_file(self.ctx, certfilebuf)
             if ret != 1:
@@ -429,8 +822,9 @@
                 else:
                     raise ssl_lib_error()
 
-            keyfilebuf = _str_to_ffi_buffer(keyfile)
-            ret = lib.SSL_CTX_use_PrivateKey_file(self.ctx, keyfilebuf,
+            ffi.errno = 0
+            buf = _str_to_ffi_buffer(keyfile)
+            ret = lib.SSL_CTX_use_PrivateKey_file(self.ctx, buf,
                                                   lib.SSL_FILETYPE_PEM)
             if ret != 1:
                 if pw_info.operationerror:
@@ -459,6 +853,116 @@
         return _SSLSocket._new__ssl_socket(self, sock, server_side,
                 server_hostname, None, None)
 
+    def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+        ffi.errno = 0
+        if cadata is None:
+            ca_file_type = -1
+        else:
+            if not isinstance(cadata, str):
+                ca_file_type = lib.SSL_FILETYPE_ASN1
+            else:
+                ca_file_type = lib.SSL_FILETYPE_PEM
+                try:
+                    cadata = cadata.encode('ascii')
+                except UnicodeEncodeError:
+                    raise TypeError("cadata should be a ASCII string or a bytes-like object")
+        if cafile is None and capath is None and cadata is None:
+            raise TypeError("cafile and capath cannot be both omitted")
+        # load from cadata
+        if cadata is not None:
+            buf = _str_to_ffi_buffer(cadata)
+            self._add_ca_certs(buf, len(buf), ca_file_type)
+
+        # load cafile or capath
+        if cafile or capath:
+            if cafile is None:
+                cafilebuf = ffi.NULL
+            else:
+                cafilebuf = _str_to_ffi_buffer(cafile)
+            if capath is None:
+                capathbuf = ffi.NULL
+            else:
+                capathbuf = _str_to_ffi_buffer(capath)
+            ret = lib.SSL_CTX_load_verify_locations(self.ctx, cafilebuf, capathbuf)
+            if ret != 1:
+                _errno = ffi.errno
+                if _errno:
+                    lib.ERR_clear_error()
+                    raise OSError(_errno, '')
+                else:
+                    raise ssl_lib_error()
+
+    def _add_ca_certs(self, data, size, ca_file_type):
+        biobuf = lib.BIO_new_mem_buf(data, size)
+        if biobuf == ffi.NULL:
+            raise ssl_error("Can't allocate buffer")
+        try:
+            store = lib.SSL_CTX_get_cert_store(self.ctx)
+            loaded = 0
+            while True:
+                if ca_file_type == lib.SSL_FILETYPE_ASN1:
+                    cert = lib.d2i_X509_bio(biobuf, ffi.NULL)
+                else:
+                    cert = lib.PEM_read_bio_X509(biobuf, ffi.NULL, ffi.NULL, ffi.NULL)
+                if not cert:
+                    break
+                try:
+                    r = lib.X509_STORE_add_cert(store, cert)
+                finally:
+                    lib.X509_free(cert)
+                if not r:
+                    err = lib.ERR_peek_last_error()
+                    if (lib.ERR_GET_LIB(err) == lib.ERR_LIB_X509 and
+                        lib.ERR_GET_REASON(err) ==
+                        lib.X509_R_CERT_ALREADY_IN_HASH_TABLE):
+                        # cert already in hash table, not an error
+                        lib.ERR_clear_error()
+                    else:
+                        break
+                loaded += 1
+
+            err = lib.ERR_peek_last_error()
+            if (ca_file_type == lib.SSL_FILETYPE_ASN1 and
+                loaded > 0 and
+                lib.ERR_GET_LIB(err) == lib.ERR_LIB_ASN1 and
+                lib.ERR_GET_REASON(err) == lib.ASN1_R_HEADER_TOO_LONG):
+                # EOF ASN1 file, not an error
+                lib.ERR_clear_error()
+            elif (ca_file_type == lib.SSL_FILETYPE_PEM and
+                  loaded > 0 and
+                  lib.ERR_GET_LIB(err) == lib.ERR_LIB_PEM and
+                  lib.ERR_GET_REASON(err) == lib.PEM_R_NO_START_LINE):
+                # EOF PEM file, not an error
+                lib.ERR_clear_error()
+            else:
+                raise ssl_lib_error()
+        finally:
+            lib.BIO_free(biobuf)
+
+    def cert_store_stats(self):
+        store = lib.SSL_CTX_get_cert_store(self.ctx)
+        x509 = 0
+        x509_ca = 0
+        crl = 0
+        objs = store.objs
+        count = lib.sk_X509_OBJECT_num(objs)
+        for i in range(count):
+            obj = lib.sk_X509_OBJECT_value(objs, i)
+            _type = lib.Cryptography_X509_OBJECT_get_type(obj)
+            if _type == lib.Cryptography_X509_LU_X509:
+                x509 += 1
+                cert = lib.Cryptography_X509_OBJECT_data_x509(obj)
+                if lib.X509_check_ca(cert):
+                    x509_ca += 1
+            elif _type == lib.Cryptography_X509_LU_CRL:
+                crl += 1
+            else:
+                # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
+                # As far as I can tell they are internal states and never
+                # stored in a cert store
+                pass
+        return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
+
 
 #    def _finalize_(self):
 #        ctx = self.ctx
@@ -472,17 +976,17 @@
 #        self = space.allocate_instance(SSLContext, w_subtype)
 #        self.__init__(space, protocol)
 #        return space.wrap(self)
-#    def session_stats_w(self, space):
-#        w_stats = space.newdict()
-#        for name, ssl_func in SSL_CTX_STATS:
-#            w_value = space.wrap(ssl_func(self.ctx))
-#            space.setitem_str(w_stats, name, w_value)
-#        return w_stats
-#
-#    def descr_set_default_verify_paths(self, space):
-#        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
-#            raise ssl_error(space, "")
-#
+
+    def session_stats(self):
+        stats = {}
+        for name, ssl_func in SSL_CTX_STATS:
+            stats[name] = ssl_func(self.ctx)
+        return stats
+
+    def set_default_verify_paths(self):
+        if not lib.SSL_CTX_set_default_verify_paths(self.ctx):
+            raise ssl_error("")
+
 #    def descr_get_options(self, space):
 #        return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
 #
@@ -500,51 +1004,6 @@
 #        if set:
 #            libssl_SSL_CTX_set_options(self.ctx, set)
 #
-#    def descr_get_verify_mode(self, space):
-#        mode = libssl_SSL_CTX_get_verify_mode(self.ctx)
-#        if mode == SSL_VERIFY_NONE:
-#            return space.newlong(PY_SSL_CERT_NONE)
-#        elif mode == SSL_VERIFY_PEER:
-#            return space.newlong(PY_SSL_CERT_OPTIONAL)
-#        elif mode == SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
-#            return space.newlong(PY_SSL_CERT_REQUIRED)
-#        raise ssl_error(space, "invalid return value from SSL_CTX_get_verify_mode")
-#
-#    def descr_set_verify_mode(self, space, w_mode):
-#        n = space.int_w(w_mode)
-#        if n == PY_SSL_CERT_NONE:
-#            mode = SSL_VERIFY_NONE
-#        elif n == PY_SSL_CERT_OPTIONAL:
-#            mode = SSL_VERIFY_PEER
-#        elif n == PY_SSL_CERT_REQUIRED:
-#            mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT
-#        else:
-#            raise oefmt(space.w_ValueError,
-#                        "invalid value for verify_mode")
-#        if mode == SSL_VERIFY_NONE and self.check_hostname:
-#            raise oefmt(space.w_ValueError,
-#                        "Cannot set verify_mode to CERT_NONE when "
-#                        "check_hostname is enabled.")
-#        libssl_SSL_CTX_set_verify(self.ctx, mode, None)
-#
-#    def descr_get_verify_flags(self, space):
-#        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-#        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
-#        return space.wrap(flags)
-#
-#    def descr_set_verify_flags(self, space, w_obj):
-#        new_flags = space.int_w(w_obj)
-#        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-#        flags = libssl_X509_VERIFY_PARAM_get_flags(store[0].c_param)
-#        flags_clear = flags & ~new_flags
-#        flags_set = ~flags & new_flags
-#        if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-#                store[0].c_param, flags_clear):
-#            raise _ssl_seterror(space, None, 0)
-#        if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-#                store[0].c_param, flags_set):
-#            raise _ssl_seterror(space, None, 0)
-#
 #    def descr_get_check_hostname(self, space):
 #        return space.newbool(self.check_hostname)
 #
@@ -556,126 +1015,35 @@
 #                        "CERT_OPTIONAL or CERT_REQUIRED")
 #        self.check_hostname = check_hostname
 #
-#    @unwrap_spec(filepath=str)
-#    def load_dh_params_w(self, space, filepath):
-#        bio = libssl_BIO_new_file(filepath, "r")
-#        if not bio:
-#            errno = get_saved_errno()
-#            libssl_ERR_clear_error()
-#            raise wrap_oserror(space, OSError(errno, ''),
-#                               exception_name = 'w_IOError')
-#        try:
-#            dh = libssl_PEM_read_bio_DHparams(bio, None, None, None)
-#        finally:
-#            libssl_BIO_free(bio)
-#        if not dh:
-#            errno = get_saved_errno()
-#            if errno != 0:
-#                libssl_ERR_clear_error()
-#                raise wrap_oserror(space, OSError(errno, ''))
-#            else:
-#                raise _ssl_seterror(space, None, 0)
-#        try:
-#            if libssl_SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
-#                raise _ssl_seterror(space, None, 0)
-#        finally:
-#            libssl_DH_free(dh)
-#
-#    def load_verify_locations_w(self, space, w_cafile=None, w_capath=None,
-#                                w_cadata=None):
-#        if space.is_none(w_cafile):
-#            cafile = None
-#        else:
-#            cafile = space.str_w(w_cafile)
-#        if space.is_none(w_capath):
-#            capath = None
-#        else:
-#            capath = space.str_w(w_capath)
-#        if space.is_none(w_cadata):
-#            cadata = None
-#            ca_file_type = -1
-#        else:
-#            if not space.isinstance_w(w_cadata, space.w_unicode):
-#                ca_file_type = SSL_FILETYPE_ASN1
-#                cadata = space.bufferstr_w(w_cadata)
-#            else:
-#                ca_file_type = SSL_FILETYPE_PEM
-#                try:
-#                    cadata = space.unicode_w(w_cadata).encode('ascii')
-#                except UnicodeEncodeError:
-#                    raise oefmt(space.w_TypeError,
-#                                "cadata should be a ASCII string or a "
-#                                "bytes-like object")
-#        if cafile is None and capath is None and cadata is None:
-#            raise oefmt(space.w_TypeError,
-#                        "cafile and capath cannot be both omitted")
-#        # load from cadata
-#        if cadata is not None:
-#            with rffi.scoped_nonmovingbuffer(cadata) as buf:
-#                self._add_ca_certs(space, buf, len(cadata), ca_file_type)
-#
-#        # load cafile or capath
-#        if cafile is not None or capath is not None:
-#            ret = libssl_SSL_CTX_load_verify_locations(
-#                self.ctx, cafile, capath)
-#            if ret != 1:
-#                errno = get_saved_errno()
-#                if errno:
-#                    libssl_ERR_clear_error()
-#                    raise wrap_oserror(space, OSError(errno, ''),
-#                                       exception_name = 'w_IOError')
-#                else:
-#                    raise _ssl_seterror(space, None, -1)
-#
-#    def _add_ca_certs(self, space, data, size, ca_file_type):
-#        biobuf = libssl_BIO_new_mem_buf(data, size)
-#        if not biobuf:
-#            raise ssl_error(space, "Can't allocate buffer")
-#        try:
-#            store = libssl_SSL_CTX_get_cert_store(self.ctx)
-#            loaded = 0
-#            while True:
-#                if ca_file_type == SSL_FILETYPE_ASN1:
-#                    cert = libssl_d2i_X509_bio(
-#                        biobuf, None)
-#                else:
-#                    cert = libssl_PEM_read_bio_X509(
-#                        biobuf, None, None, None)
-#                if not cert:
-#                    break
-#                try:
-#                    r = libssl_X509_STORE_add_cert(store, cert)
-#                finally:
-#                    libssl_X509_free(cert)
-#                if not r:
-#                    err = libssl_ERR_peek_last_error()
-#                    if (libssl_ERR_GET_LIB(err) == ERR_LIB_X509 and
-#                        libssl_ERR_GET_REASON(err) ==
-#                        X509_R_CERT_ALREADY_IN_HASH_TABLE):
-#                        # cert already in hash table, not an error
-#                        libssl_ERR_clear_error()
-#                    else:
-#                        break
-#                loaded += 1
-#
-#            err = libssl_ERR_peek_last_error()
-#            if (ca_file_type == SSL_FILETYPE_ASN1 and
-#                loaded > 0 and
-#                libssl_ERR_GET_LIB(err) == ERR_LIB_ASN1 and
-#                libssl_ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG):
-#                # EOF ASN1 file, not an error
-#                libssl_ERR_clear_error()
-#            elif (ca_file_type == SSL_FILETYPE_PEM and
-#                  loaded > 0 and
-#                  libssl_ERR_GET_LIB(err) == ERR_LIB_PEM and
-#                  libssl_ERR_GET_REASON(err) == PEM_R_NO_START_LINE):
-#                # EOF PEM file, not an error
-#                libssl_ERR_clear_error()
-#            else:
-#                raise _ssl_seterror(space, None, 0)
-#        finally:
-#            libssl_BIO_free(biobuf)
-#
+    def load_dh_params(self, filepath):
+        ffi.errno = 0
+        if filepath is None:
+            raise TypeError("filepath must not be None")
+        buf = _str_to_ffi_buffer(filepath, zeroterm=True)
+        mode = ffi.new("char[]",b"r")
+        ffi.errno = 0
+        bio = lib.BIO_new_file(buf, mode)
+        if bio == ffi.NULL:
+            _errno = ffi.errno
+            lib.ERR_clear_error()
+            raise OSError(_errno, '')
+        try:
+            dh = lib.PEM_read_bio_DHparams(bio, ffi.NULL, ffi.NULL, ffi.NULL)
+        finally:
+            lib.BIO_free(bio)
+        if dh == ffi.NULL:
+            _errno = ffi.errno
+            if _errno != 0:
+                lib.ERR_clear_error()
+                raise OSError(_errno, '')
+            else:
+                raise ssl_lib_error()
+        try:
+            if lib.SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
+                raise ssl_lib_error()
+        finally:
+            lib.DH_free(dh)
+
 #    def cert_store_stats_w(self, space):
 #        store = libssl_SSL_CTX_get_cert_store(self.ctx)
 #        x509 = 0
@@ -720,63 +1088,126 @@
 #
 #        self.alpn_protocols = SSLAlpnProtocols(self.ctx, protos)
 #
-#    def get_ca_certs_w(self, space, w_binary_form=None):
-#        if w_binary_form and space.is_true(w_binary_form):
-#            binary_mode = True
-#        else:
-#            binary_mode = False
-#        rlist = []
-#        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-#        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-#            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-#            if intmask(obj.c_type) != X509_LU_X509:
-#                # not a x509 cert
-#                continue
-#            # CA for any purpose
-#            cert = libssl_pypy_X509_OBJECT_data_x509(obj)
-#            if not libssl_X509_check_ca(cert):
-#                continue

From pypy.commits at gmail.com  Mon Nov 14 08:29:15 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 14 Nov 2016 05:29:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: merge updates to py3.5
Message-ID: <5829bc2b.058a1c0a.3d288.afd3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88374:2666380e3fad
Date: 2016-11-14 14:28 +0100
http://bitbucket.org/pypy/pypy/changeset/2666380e3fad/

Log:	merge updates to py3.5

diff too long, truncating to 2000 out of 17845 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -2,6 +2,7 @@
 from ctypes.test import need_symbol, xfail
 import unittest
 import os
+import sys
 
 import ctypes
 import _ctypes_test
@@ -279,7 +280,6 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
 
-    @xfail
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
         # Issue #23319
@@ -295,5 +295,11 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
 
+    # see issue #1213, on big endian it fails for the little endian case
+    if sys.byteorder == 'little':
+        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
+    elif sys.byteorder == 'big':
+        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -745,8 +745,7 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-
-            except (OSError, ValueError):
+            except:
                 self.close()
                 raise
 
diff --git a/lib-python/3/test/test_bytes.py b/lib-python/3/test/test_bytes.py
--- a/lib-python/3/test/test_bytes.py
+++ b/lib-python/3/test/test_bytes.py
@@ -1006,9 +1006,13 @@
     def test_del_expand(self):
         # Reducing the size should not expand the buffer (issue #23985)
         b = bytearray(10)
-        size = sys.getsizeof(b)
-        del b[:1]
-        self.assertLessEqual(sys.getsizeof(b), size)
+        try:
+            size = sys.getsizeof(b)
+        except TypeError:
+            pass            # e.g. on pypy
+        else:
+            del b[:1]
+            self.assertLessEqual(sys.getsizeof(b), size)
 
     def test_extended_set_del_slice(self):
         indices = (0, None, 1, 3, 19, 300, 1<<333, -1, -2, -31, -300)
diff --git a/lib-python/3/test/test_collections.py b/lib-python/3/test/test_collections.py
--- a/lib-python/3/test/test_collections.py
+++ b/lib-python/3/test/test_collections.py
@@ -13,6 +13,7 @@
 import re
 import sys
 import types
+import gc
 from collections import UserDict, UserString, UserList
 from collections import ChainMap
 from collections import deque
@@ -1630,7 +1631,7 @@
 ################################################################################
 
 py_coll = import_fresh_module('collections', blocked=['_collections'])
-c_coll = import_fresh_module('collections', fresh=['_collections'])
+c_coll = import_fresh_module('_collections', fresh=['_collections'])
 
 
 @contextlib.contextmanager
@@ -2052,7 +2053,9 @@
             obj = MyOD([(None, obj)])
             obj.i = i
         del obj
-        support.gc_collect()
+        # PyPy change: we only collect 1 MyOD instance per GC
+        for _ in range(100):
+            gc.collect()
         self.assertEqual(deleted, list(reversed(range(100))))
 
     def test_delitem_hash_collision(self):
@@ -2212,11 +2215,11 @@
     OrderedDict = py_coll.OrderedDict
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase):
 
     module = c_coll
-    OrderedDict = c_coll.OrderedDict
+    OrderedDict = getattr(c_coll, 'OrderedDict', None)
 
     def test_key_change_during_iteration(self):
         OrderedDict = self.OrderedDict
@@ -2246,8 +2249,10 @@
 class CPythonOrderedDictSubclassTests(CPythonOrderedDictTests):
 
     module = c_coll
-    class OrderedDict(c_coll.OrderedDict):
-        pass
+
+    if hasattr(c_coll, 'OrderedDict'):
+        class OrderedDict(c_coll.OrderedDict):
+            pass
 
 
 class PurePythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
@@ -2261,7 +2266,7 @@
         self.assertRaises(KeyError, d.popitem)
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
 
     @classmethod
@@ -2286,7 +2291,7 @@
         self.assertRaises(KeyError, d.popitem)
 
 
- at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+ at unittest.skipUnless(hasattr(c_coll, 'OrderedDict'), 'requires the C version of the collections module')
 class CPythonSubclassMappingTests(mapping_tests.BasicTestMappingProtocol):
 
     @classmethod
diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py
--- a/lib-python/3/test/test_functools.py
+++ b/lib-python/3/test/test_functools.py
@@ -129,6 +129,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -229,13 +230,7 @@
                 raise IndexError
 
         f = self.partial(object)
-        if support.check_impl_detail(pypy=True):
-            # CPython fails, pypy does not :-)
-            f.__setstate__(BadSequence())
-        else:
-            self.assertRaisesRegex(SystemError,
-                    "new style getargs format but argument is not a tuple",
-                    f.__setstate__, BadSequence())
+        self.assertRaises(TypeError, f.__setstate__, BadSequence())
 
 
 class TestPartialPy(TestPartial, unittest.TestCase):
diff --git a/lib-python/3/test/test_generators.py b/lib-python/3/test/test_generators.py
--- a/lib-python/3/test/test_generators.py
+++ b/lib-python/3/test/test_generators.py
@@ -96,8 +96,8 @@
         # generator names must be a string and cannot be deleted
         self.assertRaises(TypeError, setattr, gen, '__name__', 123)
         self.assertRaises(TypeError, setattr, gen, '__qualname__', 123)
-        self.assertRaises(TypeError, delattr, gen, '__name__')
-        self.assertRaises(TypeError, delattr, gen, '__qualname__')
+        self.assertRaises((TypeError, AttributeError), delattr, gen, '__name__')
+        self.assertRaises((TypeError, AttributeError), delattr, gen, '__qualname__')
 
         # modify names of the function creating the generator
         func.__qualname__ = "func_qualname"
diff --git a/lib-python/3/test/test_httpservers.py b/lib-python/3/test/test_httpservers.py
--- a/lib-python/3/test/test_httpservers.py
+++ b/lib-python/3/test/test_httpservers.py
@@ -254,7 +254,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('GET', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         self.assertTrue(
             err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n'))
@@ -265,7 +266,8 @@
 
         with support.captured_stderr() as err:
             self.con.request('ERROR', '/')
-            self.con.getresponse()
+            with self.con.getresponse():
+                pass
 
         lines = err.getvalue().split('\n')
         self.assertTrue(lines[0].endswith('code 404, message File not found'))
diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py
--- a/lib-python/3/test/test_logging.py
+++ b/lib-python/3/test/test_logging.py
@@ -1665,7 +1665,7 @@
         logger = logging.getLogger("http")
         root_logger = self.root_logger
         root_logger.removeHandler(self.root_logger.handlers[0])
-        for secure in (False, True):
+        for secure in (False,):  # XXX: disable SSL tests until it works
             addr = ('localhost', 0)
             if secure:
                 try:
diff --git a/lib-python/3/test/test_sax.py b/lib-python/3/test/test_sax.py
--- a/lib-python/3/test/test_sax.py
+++ b/lib-python/3/test/test_sax.py
@@ -16,7 +16,6 @@
 from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl
 from io import BytesIO, StringIO
 import codecs
-import gc
 import os.path
 import shutil
 from test import support
@@ -180,7 +179,7 @@
             with self.assertRaises(SAXException):
                 self.check_parse(TESTFN)
             # Collect leaked file.
-            gc.collect()
+            support.gc_collect()
         with open(TESTFN, 'rb') as f:
             with self.assertRaises(SAXException):
                 self.check_parse(f)
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -1898,7 +1898,14 @@
                 else:
                     self.sock.close()
 
+            # PyPy change
             def run(self):
+                try:
+                    self._run()
+                finally:
+                    self.close()
+
+            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
diff --git a/lib-python/3/test/test_urllib2_localnet.py b/lib-python/3/test/test_urllib2_localnet.py
--- a/lib-python/3/test/test_urllib2_localnet.py
+++ b/lib-python/3/test/test_urllib2_localnet.py
@@ -544,6 +544,7 @@
         self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"])
 
     def test_https(self):
+        self.skipTest('Segfaults on PyPy')
         handler = self.start_https_server()
         context = ssl.create_default_context(cafile=CERT_localhost)
         data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context)
@@ -573,6 +574,7 @@
                          cadefault=True)
 
     def test_https_sni(self):
+        self.skipTest('Segfaults on PyPy')
         if ssl is None:
             self.skipTest("ssl module required")
         if not ssl.HAS_SNI:
diff --git a/lib-python/3/tkinter/test/test_tkinter/test_images.py b/lib-python/3/tkinter/test/test_tkinter/test_images.py
--- a/lib-python/3/tkinter/test/test_tkinter/test_images.py
+++ b/lib-python/3/tkinter/test/test_tkinter/test_images.py
@@ -36,6 +36,7 @@
         self.assertEqual(image.height(), 16)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def test_create_from_data(self):
@@ -50,6 +51,7 @@
         self.assertEqual(image.height(), 16)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def assertEqualStrList(self, actual, expected):
@@ -130,6 +132,7 @@
         self.assertEqual(image['file'], testfile)
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def check_create_from_data(self, ext):
@@ -147,6 +150,7 @@
         self.assertEqual(image['file'], '')
         self.assertIn('::img::test', self.root.image_names())
         del image
+        support.gc_collect()
         self.assertNotIn('::img::test', self.root.image_names())
 
     def test_create_from_ppm_file(self):
diff --git a/lib-python/3/tkinter/test/test_ttk/test_extensions.py b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
--- a/lib-python/3/tkinter/test/test_ttk/test_extensions.py
+++ b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
@@ -2,7 +2,7 @@
 import unittest
 import tkinter
 from tkinter import ttk
-from test.support import requires, run_unittest, swap_attr
+from test.support import requires, run_unittest, swap_attr, gc_collect
 from tkinter.test.support import AbstractTkTest, destroy_default_root
 
 requires('gui')
diff --git a/lib-python/3/tkinter/test/test_ttk/test_widgets.py b/lib-python/3/tkinter/test/test_ttk/test_widgets.py
--- a/lib-python/3/tkinter/test/test_ttk/test_widgets.py
+++ b/lib-python/3/tkinter/test/test_ttk/test_widgets.py
@@ -1,7 +1,7 @@
 import unittest
 import tkinter
 from tkinter import ttk, TclError
-from test.support import requires
+from test.support import requires, gc_collect
 import sys
 
 from tkinter.test.test_ttk.test_functions import MockTclObj
@@ -837,6 +837,7 @@
         self.assertEqual(conv(self.scale.get()), var.get())
         self.assertEqual(conv(self.scale.get()), max + 5)
         del var
+        gc_collect()
 
         # the same happens with the value option
         self.scale['value'] = max + 10
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -42,6 +42,13 @@
         self, func, args = args[0], args[1], args[2:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
+        if isinstance(func, partial):
+            args = func._args + args
+            tmpkw = func._keywords.copy()
+            tmpkw.update(keywords)
+            keywords = tmpkw
+            del tmpkw
+            func = func._func
         self._func = func
         self._args = args
         self._keywords = keywords
@@ -113,3 +120,24 @@
             self.__dict__.clear()
         else:
             self.__dict__.update(d)
+
+
+ at builtinify
+def cmp_to_key(mycmp):
+    """Convert a cmp= function into a key= function"""
+    class K(object):
+        __slots__ = ['obj']
+        def __init__(self, obj):
+            self.obj = obj
+        def __lt__(self, other):
+            return mycmp(self.obj, other.obj) < 0
+        def __gt__(self, other):
+            return mycmp(self.obj, other.obj) > 0
+        def __eq__(self, other):
+            return mycmp(self.obj, other.obj) == 0
+        def __le__(self, other):
+            return mycmp(self.obj, other.obj) <= 0
+        def __ge__(self, other):
+            return mycmp(self.obj, other.obj) >= 0
+        __hash__ = None
+    return K
diff --git a/lib_pypy/_libmpdec/io.c b/lib_pypy/_libmpdec/io.c
--- a/lib_pypy/_libmpdec/io.c
+++ b/lib_pypy/_libmpdec/io.c
@@ -446,7 +446,7 @@
 
     if (mpd_isspecial(dec)) {
 
-        mem = sizeof "-Infinity";
+        mem = sizeof "-Infinity%";
         if (mpd_isnan(dec) && dec->len > 0) {
             /* diagnostic code */
             mem += dec->digits;
@@ -609,10 +609,10 @@
             *cp++ = (flags&MPD_FMT_UPPER) ? 'E' : 'e';
             cp = exp_to_string(cp, ldigits-dplace);
         }
+    }
 
-        if (flags&MPD_FMT_PERCENT) {
-            *cp++ = '%';
-        }
+    if (flags&MPD_FMT_PERCENT) {
+        *cp++ = '%';
     }
 
     assert(cp < decstring+mem);
@@ -1260,6 +1260,9 @@
             stackspec.align = '>';
             spec = &stackspec;
         }
+        if (type == '%') {
+            flags |= MPD_FMT_PERCENT;
+        }
     }
     else {
         uint32_t workstatus = 0;
diff --git a/lib_pypy/_libmpdec/mpdecimal.c b/lib_pypy/_libmpdec/mpdecimal.c
--- a/lib_pypy/_libmpdec/mpdecimal.c
+++ b/lib_pypy/_libmpdec/mpdecimal.c
@@ -43,6 +43,7 @@
 #ifdef PPRO
   #if defined(_MSC_VER)
     #include <float.h>
+    #pragma float_control(precise, on)
     #pragma fenv_access(on)
   #elif !defined(__OpenBSD__) && !defined(__NetBSD__)
     /* C99 */
diff --git a/lib_pypy/_libmpdec/mpdecimal.h b/lib_pypy/_libmpdec/mpdecimal.h
--- a/lib_pypy/_libmpdec/mpdecimal.h
+++ b/lib_pypy/_libmpdec/mpdecimal.h
@@ -108,9 +108,13 @@
 
 #define MPD_MAJOR_VERSION 2
 #define MPD_MINOR_VERSION 4
-#define MPD_MICRO_VERSION 0
+#define MPD_MICRO_VERSION 1
 
-#define MPD_VERSION "2.4.0"
+#define MPD_VERSION "2.4.1"
+
+#define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \
+                         (MPD_MINOR_VERSION << 16) | \
+                         (MPD_MICRO_VERSION <<  8))
 
 const char *mpd_version(void);
 
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -359,7 +359,7 @@
             for i in range(objc[0]):
                 result.append(FromObj(self, objv[0][i]))
             return tuple(result)
-        elif isinstance(arg, tuple):
+        elif isinstance(arg, (tuple, list)):
             return self._splitObj(arg)
         if isinstance(arg, str):
             arg = arg.encode('utf-8')
@@ -378,6 +378,8 @@
             return tuple(result)
         elif isinstance(arg, tuple):
             return arg
+        elif isinstance(arg, list):
+            return tuple(arg)
         elif isinstance(arg, str):
             arg = arg.encode('utf8')
 
@@ -411,6 +413,9 @@
                 result[i] = newelem
             if result is not None:
                 return tuple(result)
+        if isinstance(arg, list):
+            # Recursively invoke SplitObj for all list items.
+            return tuple(self._splitObj(elem) for elem in arg)
         elif isinstance(arg, str):
             argc = tkffi.new("int*")
             argv = tkffi.new("char***")
@@ -483,7 +488,10 @@
     def getboolean(self, s):
         if isinstance(s, int):
             return bool(s)
-        s = s.encode('utf-8')
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
+            raise TypeError
         if b'\x00' in s:
             raise TypeError
         v = tkffi.new("int*")
@@ -493,9 +501,12 @@
         return bool(v[0])
 
     def getint(self, s):
-        if isinstance(s, (int, long)):
+        if isinstance(s, int):
             return s
-        s = s.encode('utf-8')
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
+            raise TypeError
         if b'\x00' in s:
             raise TypeError
         if tklib.HAVE_LIBTOMMATH or tklib.HAVE_WIDE_INT_TYPE:
@@ -517,11 +528,14 @@
             return v[0]
 
     def getdouble(self, s):
-        if isinstance(s, float):
-            return s
-        if '\x00' in s:
+        if isinstance(s, (float, int)):
+            return float(s)
+        try:
+            s = s.encode('utf-8')
+        except AttributeError:
             raise TypeError
-        s = s.encode('utf-8')
+        if b'\x00' in s:
+            raise TypeError
         v = tkffi.new("double*")
         res = tklib.Tcl_GetDouble(self.interp, s, v)
         if res == tklib.TCL_ERROR:
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -70,7 +70,7 @@
 
 def AsBignumObj(value):
     sign = -1 if value < 0 else 1
-    hexstr = '%x' % abs(value)
+    hexstr = b'%x' % abs(value)
     bigValue = tkffi.new("mp_int*")
     tklib.mp_init(bigValue)
     try:
@@ -133,7 +133,7 @@
 
 def AsObj(value):
     if isinstance(value, bytes):
-        return tklib.Tcl_NewStringObj(value, len(value))
+        return tklib.Tcl_NewByteArrayObj(value, len(value))
     if isinstance(value, bool):
         return tklib.Tcl_NewBooleanObj(value)
     if isinstance(value, int):
@@ -151,7 +151,7 @@
                 return AsBignumObj(value)
     if isinstance(value, float):
         return tklib.Tcl_NewDoubleObj(value)
-    if isinstance(value, tuple):
+    if isinstance(value, (tuple, list)):
         argv = tkffi.new("Tcl_Obj*[]", len(value))
         for i in range(len(value)):
             argv[i] = AsObj(value[i])
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.0"
+__version_info__ = (1, 9, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -9,8 +9,8 @@
   Philip Jenvey
   Ronan Lamy
   Brian Kearns
+  Richard Plangger
   Michael Hudson
-  Richard Plangger
   Manuel Jacob
   David Schneider
   Holger Krekel
@@ -54,10 +54,10 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
-  Spenser Bauman
   Tyler Wade
   Vincent Legoll
   Michael Foord
@@ -65,13 +65,13 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Stefano Rivera
   Timo Paulssen
   Edd Barrett
   Squeaky
@@ -89,12 +89,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -128,10 +128,11 @@
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -147,17 +148,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
   Daniel Patrick
-  Jasper Schulz
-  Christian Hudon
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -165,8 +165,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -175,11 +175,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -196,7 +197,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -219,6 +219,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -231,6 +232,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -258,9 +260,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -274,10 +274,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
-  "Aaron Gallagher
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -287,8 +287,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -302,13 +303,11 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
-  JohnDoe
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
   reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
@@ -316,25 +315,24 @@
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
   remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
   werat
-  hgattic
+
+
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -407,3 +407,18 @@
 long and gnarly. On the other hand, all our infrastructure (buildbots,
 benchmarking, etc) would have to be adapted. So unless somebody steps up and
 volunteers to do all that work, it will likely not happen.
+
+
+What is needed for Windows 64 support of PyPy?
+-----------------------------------------------
+
+First, please note that the Windows 32 PyPy binary works just fine on Windows
+64. The only problem is that it only supports up to 4GB of heap per process.
+
+As to real Windows 64 support: Currently we don't have an active PyPy developer
+whose main development platform is Windows. So if you are interested in getting
+Windows 64 support, we encourage you to volunteer `to make it happen`_! Another
+option would be to pay some PyPy developers to implement Windows 64 support,
+but so far there doesn't seem to be an overwhelming commercial interest in it.
+
+.. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -1,4 +1,4 @@
-The PyPy Release Process
+PyPy's Release Process
 ========================
 
 Release Policy
@@ -39,41 +39,60 @@
   Some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
   necessary.
-* update pypy/doc/contributor.rst (and possibly LICENSE)
-  pypy/doc/tool/makecontributor.py generates the list of contributors
-* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  create a fresh whatsnew_head.rst after the release
-  and add the new file to  pypy/doc/index-of-whatsnew.rst
-* go to pypy/tool/release and run
-  ``force-builds.py <release branch>``
-  The following JIT binaries should be built, however, we need more buildbots
-  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
-  freebsd64 
 
-* wait for builds to complete, make sure there are no failures
-* download the builds, repackage binaries. Tag the release version
-  and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+* Update and write documentation
 
-  Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
-  and to cobra, as some packagers prefer a clearly labeled source package
-  ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
-  unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+  * update pypy/doc/contributor.rst (and possibly LICENSE)
+    pypy/doc/tool/makecontributor.py generates the list of contributors
 
-* Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+  * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+    create a fresh whatsnew_head.rst after the release
+    and add the new file to  pypy/doc/index-of-whatsnew.rst
 
-* write release announcement pypy/doc/release-x.y(.z).txt
+  * write release announcement pypy/doc/release-VERSION.rst
+    The release announcement should contain a direct link to the download page
 
-  The release announcement should contain a direct link to the download page
+  * Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
 
-* Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
+* Build and upload the release tar-balls
 
-* update pypy.org (under extradoc/pypy.org), rebuild and commit
+  * go to pypy/tool/release and run
+    ``force-builds.py <release branch>``
+    The following JIT binaries should be built, however, we need more buildbots
+    windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+    freebsd64 
 
-* post announcement on morepypy.blogspot.com
-* send announcements to twitter.com, pypy-dev, python-list,
-  python-announce, python-dev ...
+  * wait for builds to complete, make sure there are no failures
 
-* add a tag on the pypy/jitviewer repo that corresponds to pypy release
-* add a tag on the codespeed web site that corresponds to pypy release
-* revise versioning at https://readthedocs.org/projects/pypy
+  * send out a mailing list message asking for people to test before uploading
+    to prevent having to upload more than once
+
+  * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
+    that the source tarball can be produced in the next steps
+
+  * download the builds, repackage binaries. Tag the release version
+    and download and repackage source from bitbucket. You may find it
+    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+
+    Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
+    and to cobra, as some packagers prefer a clearly labeled source package
+    ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
+    unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
+  * Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+
+* Send out a mailing list message asking for last-minute comments and testing
+
+* RELEASE !  
+
+  * update pypy.org (under extradoc/pypy.org), rebuild and commit, using the
+    hashes produced from the ``repackage.sh`` script or by hand
+
+  * post announcement on morepypy.blogspot.com
+  * send announcements to twitter.com, pypy-dev, python-list,
+    python-announce, python-dev ...
+
+* If all is OK, document the released version
+
+  * add a tag on the codespeed web site that corresponds to pypy release
+  * revise versioning at https://readthedocs.org/projects/pypy
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -6,17 +6,24 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
 
 We continue to make incremental improvements to our C-API
-compatability layer (cpyext). We pass all but a few of the tests in the
+compatibility layer (cpyext). We pass all but a few of the tests in the
 upstream numpy `test suite`_. 
 
 Work proceeds at a good pace on the PyPy3.5
 version due to a grant_ from the Mozilla Foundation, and some of those
-changes have been backported to PyPy2.7 where relevant
+changes have been backported to PyPy2.7 where relevant.
+
+The PowerPC and s390x backend have been enhanced_ with the capability use SIMD instructions
+ for micronumpy loops.
 
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
-XXX
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
+well as maintaining support for previous versions.
+
+CFFI_ has been updated to 1.9, improving an already great package for
+interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -35,12 +42,13 @@
 with making RPython's JIT even better.
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
-.. _cffi: https://cffi.readthedocs.org
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+.. _`enhanced`: https://morepypy.blogspot.co.at/2016/11/vectorization-extended-powerpc-and-s390x.html
 
 What is PyPy?
 =============
@@ -64,42 +72,46 @@
   * **s390x** running Linux
 
 .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
-.. _`dynamic languages`: http://pypyjs.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
 
 Other Highlights (since 5.4 released Aug 31, 2016)
 =========================================================
 
 * New features
+
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
-  * Backport fixes to rposix on windows from py2.5
+  * Backport fixes to rposix on windows from py3.5
   * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
   * Add ``inode`` to ``scandir()`` on posix systems
   * Support more attributes on ``super``
   * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
   * Restore the ability to translate with CPython
-  * Update to CFFI 1.8.4
+  * Update to CFFI 1.9.0
   * Support the new buffer protocol in cpyext and numpypy
   * Add ``rposix.sync()``
   * Support full-precision nanosecond times in os.stat()
   * Add documentation about the assembler backends to RPYthon
   * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
-    changes downstream packaging requirments
+    changes downstream packaging requirements
   * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
   * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
     be ``str`` or ``unicode``)
   * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
   * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
     relevant platforms
-  * Backport the ``'faulthandler`` module from py3.5
+  * Backport the ``faulthandler`` module from py3.5
   * Improve the error message when trying to call a method where the ``self``
     parameter is missing in the definition
   * Implement ``rposix.cpu_count``
   * Support translation on FreeBSD running on PowerPC
   * Implement ``__rmod__`` on ``str`` and ``unicode`` types
   * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
+
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
   * Fix translation of the sandbox
   * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
@@ -120,6 +132,7 @@
   * ``_numpypy.add.reduce`` returns a scalar now
 
 * Performance improvements:
+
   * Improve method calls on oldstyle classes
   * Clean and refactor code for testing cpyext to allow sharing with py3.5
   * Refactor a building the map of reflected ops in ``_numpypy``
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -111,4 +111,4 @@
         "hg log -r %s --template '{branch}'" % startrev)
     if errcode != 0:
         py.test.skip('no Mercurial repo')
-    assert wc_branch == 'default'
+    assert wc_branch in ('default', "'default'") # sometimes the ' leaks (windows)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -32,7 +32,7 @@
     'Niklaus Haldimann': ['nik'],
     'Alexander Schremmer': ['xoraxax'],
     'Anders Hammarquist': ['iko'],
-    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'David Edelsohn': ['edelsoh', 'edelsohn','opassembler.py'],
     'Niko Matsakis': ['niko'],
     'Jakub Gustak': ['jlg'],
     'Guido Wesdorp': ['guido'],
@@ -75,6 +75,9 @@
     'Spenser Bauman':['Spenser Andrew Bauman'],
     'Raffael Tfirst':['raffael.tfirst at gmail.com'],
     'timo':['timo at eistee.fritz.box'],
+    'Jasper Schulz':['Jasper.Schulz'],
+    'Aaron Gallagher':['"Aaron Gallagher'],
+    'Yasir Suhail':['yasirs'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,9 +1,9 @@
 ==========================
-What's new in PyPy2.7 5.4+
+What's new in PyPy2.7 5.6+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 522736f816dc
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
 
 .. branch: mappingproxy
 .. branch: py3k-finish_time
@@ -14,75 +14,8 @@
 .. branch: rpython-resync
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
-.. branch: buffer-interface
-Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
-in numpypy
+.. branch: rpython-error-to-systemerror
 
-.. branch: force-virtual-state
-Improve merging of virtual states in the JIT in order to avoid jumping to the
-preamble. Accomplished by allocating virtual objects where non-virtuals are
-expected.
-
-.. branch: conditional_call_value_3
-JIT residual calls: if the called function starts with a fast-path
-like "if x.foo != 0: return x.foo", then inline the check before
-doing the CALL.  For now, string hashing is about the only case.
-
-.. branch: search-path-from-libpypy
-
-The compiled pypy now looks for its lib-python/lib_pypy path starting
-from the location of the *libpypy-c* instead of the executable. This is
-arguably more consistent, and also it is what occurs anyway if you're
-embedding pypy.  Linux distribution packagers, take note!  At a minimum,
-the ``libpypy-c.so`` must really be inside the path containing
-``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
-from somewhere else.  You no longer have to do the same with the
-``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
-
-.. branch: _warnings
-
-CPython allows warning.warn(('something', 1), Warning), on PyPy this
-produced a "expected a readable buffer object" error. Test and fix.
-
-.. branch: stricter-strip
-
-CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
-allowed as arguments. Test and fix for str and unicode
-
-.. branch: faulthandler
-
-Port the 'faulthandler' module to PyPy default.  This module is standard
-in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
-
-.. branch: test-cpyext
-
-Refactor cpyext testing to be more pypy3-friendly.
-
-.. branch: better-error-missing-self
-
-Improve the error message when the user forgot the "self" argument of a method.
-
-
-.. fb6bb835369e
-Change the ``timeit`` module: it now prints the average time and the standard
-deviation over 7 runs by default, instead of the minimum. The minimum is often
-misleading.
-
-.. branch: unrecursive-opt
-
-Make optimiseopt iterative instead of recursive so it can be reasoned about
-more easily and debugging is faster.
-
-.. branch: Tiberiumk/fix-2412-1476011166874
-.. branch: redirect-assembler-jitlog
-
-
-
-.. branch: stdlib-2.7.12
-
-Update stdlib to version 2.7.12
-
-.. branch: buffer-interface2
-
-Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
-types still missing
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -0,0 +1,113 @@
+=========================
+What's new in PyPy2.7 5.6
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
+
+.. branch: mappingproxy
+.. branch: py3k-finish_time
+.. branch: py3k-kwonly-builtin
+.. branch: py3k_add_terminal_size
+.. branch: testing-cleanup-py3k
+
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warnings
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode
+
+.. branch: faulthandler
+
+Port the 'faulthandler' module to PyPy default.  This module is standard
+in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
+
+.. branch: test-cpyext
+
+Refactor cpyext testing to be more pypy3-friendly.
+
+.. branch: better-error-missing-self
+
+Improve the error message when the user forgot the "self" argument of a method.
+
+
+.. fb6bb835369e
+Change the ``timeit`` module: it now prints the average time and the standard
+deviation over 7 runs by default, instead of the minimum. The minimum is often
+misleading.
+
+.. branch: unrecursive-opt
+
+Make optimiseopt iterative instead of recursive so it can be reasoned about
+more easily and debugging is faster.
+
+.. branch: Tiberiumk/fix-2412-1476011166874
+.. branch: redirect-assembler-jitlog
+
+
+
+.. branch: stdlib-2.7.12
+
+Update stdlib to version 2.7.12
+
+.. branch: buffer-interface2
+
+Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
+types still missing
+
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: zarch-simd-support
+
+s390x implementation for vector operations used in VecOpt
+
+.. branch: ppc-vsx-support
+
+PowerPC implementation for vector operations used in VecOpt
+
+.. branch: newinitwarn
+
+Match CPython's stricter handling of __new/init__ arguments
+
+.. branch: openssl-1.1
+
+Support for OpenSSL version 1.1 (in addition to version 1.0).
+Tested on Linux (1.1, 1.0), on Win32, and Mac (1.0 only)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -20,6 +20,22 @@
 
 .. _our downloads: http://pypy.org/download.html
 
+Installing Visual Compiler v9 (for Python 2.7)
+----------------------------------------------
+
+This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
+was checked in Nov 2016). Note that the compiler suite will be installed in
+``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
+Using a current version of ``setuptools`` will be able to find it there. For
+Windows 10, you must right-click the download, and under ``Properties`` ->
+``Compatibility`` mark it as ``Run run this program in comatibility mode for``
+``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
+otherwise ``mt.exe`` will silently fail. Installation will begin automatically
+by running the mt.exe command by hand from a DOS window (that is how the author
+discovered the problem).
+
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
 
 Translating PyPy with Visual Studio
 -----------------------------------
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -34,7 +34,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -110,18 +115,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('builtins'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -129,6 +139,9 @@
                 debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
@@ -247,7 +260,8 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if config.translation.profopt is not None:
+            if (config.translation.profopt is not None
+                    and not config.translation.noprofopt):
                 raise Exception("Cannot use the --profopt option "
                                 "when --shared is on (it is by default). "
                                 "See issue #2398.")
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -662,7 +662,7 @@
     ops.LOAD_DEREF: 1,
     ops.STORE_DEREF: -1,
     ops.DELETE_DEREF: 0,
-    
+
     ops.GET_AWAITABLE: 0,
     ops.SETUP_ASYNC_WITH: 0,
     ops.BEFORE_ASYNC_WITH: 1,
@@ -682,10 +682,10 @@
     ops.JUMP_IF_FALSE_OR_POP: 0,
     ops.POP_JUMP_IF_TRUE: -1,
     ops.POP_JUMP_IF_FALSE: -1,
-    # TODO 
+    # TODO
     ops.JUMP_IF_NOT_DEBUG: 0,
 
-    # TODO 
+    # TODO
     ops.BUILD_LIST_FROM_ARG: 1,
     # TODO
     ops.LOAD_CLASSDEREF: 1,
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -210,7 +210,9 @@
 
     def sub_scope(self, kind, name, node, lineno):
         """Convenience function for compiling a sub scope."""
-        if self.qualname:
+        if self.scope.lookup(name) == symtable.SCOPE_GLOBAL_EXPLICIT:
+            qualname = name
+        elif self.qualname:
             qualname = '%s.%s' % (self.qualname, name)
         else:
             qualname = name
@@ -578,7 +580,7 @@
         self.pop_frame_block(F_BLOCK_LOOP, start)
         self.visit_sequence(fr.orelse)
         self.use_next_block(end)
-    
+
     def visit_AsyncFor(self, fr):
         self.update_position(fr.lineno, True)
         b_try = self.new_block()
@@ -588,21 +590,21 @@
         b_try_cleanup = self.new_block()
         b_after_loop = self.new_block()
         b_after_loop_else = self.new_block()
-        
+
         self.emit_jump(ops.SETUP_LOOP, b_after_loop)
         self.push_frame_block(F_BLOCK_LOOP, b_try)
-        
+
         fr.iter.walkabout(self)
         self.emit_op(ops.GET_AITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
-        
+
         self.use_next_block(b_try)
         # This adds another line, so each for iteration can be traced.
         self.lineno_set = False
         self.emit_jump(ops.SETUP_EXCEPT, b_except)
         self.push_frame_block(F_BLOCK_EXCEPT, b_try)
-        
+
         self.emit_op(ops.GET_ANEXT)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
@@ -610,13 +612,13 @@
         self.emit_op(ops.POP_BLOCK)
         self.pop_frame_block(F_BLOCK_EXCEPT, b_try)
         self.emit_jump(ops.JUMP_FORWARD, b_after_try)
-        
+
         self.use_next_block(b_except)
         self.emit_op(ops.DUP_TOP)
         self.emit_op_name(ops.LOAD_GLOBAL, self.names, "StopAsyncIteration")
         self.emit_op_arg(ops.COMPARE_OP, 10)
         self.emit_jump(ops.POP_JUMP_IF_FALSE, b_try_cleanup, True)
-        
+
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_TOP)
@@ -630,23 +632,23 @@
         self.emit_op(ops.POP_TOP)
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.emit_jump(ops.JUMP_ABSOLUTE, b_after_loop_else, True)
-        
+
         self.use_next_block(b_try_cleanup)
         self.emit_op(ops.END_FINALLY)
-        
+
         self.use_next_block(b_after_try)
         self.visit_sequence(fr.body)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_try, True)
-        
+
         self.emit_op(ops.POP_BLOCK) # for SETUP_LOOP
         self.pop_frame_block(F_BLOCK_LOOP, b_try)
-        
+
         self.use_next_block(b_after_loop)
         self.emit_jump(ops.JUMP_ABSOLUTE, b_end, True)
-        
+
         self.use_next_block(b_after_loop_else)
         self.visit_sequence(fr.orelse)
-        
+
         self.use_next_block(b_end)
 
     def visit_While(self, wh):
@@ -701,7 +703,7 @@
             self.emit_op(ops.POP_TOP)
             if handler.name:
                 ## generate the equivalent of:
-                ## 
+                ##
                 ## try:
                 ##     # body
                 ## except type as name:
@@ -960,7 +962,7 @@
 
     def visit_AsyncWith(self, wih):
         self.update_position(wih.lineno, True)
-        self.handle_asyncwithitem(wih, 0, is_async=True)
+        self.handle_withitem(wih, 0, is_async=True)
 
     def visit_Raise(self, rais):
         self.update_position(rais.lineno, True)
@@ -1008,7 +1010,7 @@
         self.emit_op(ops.GET_YIELD_FROM_ITER)
         self.load_const(self.space.w_None)
         self.emit_op(ops.YIELD_FROM)
-    
+
     def visit_Await(self, aw):
         self.update_position(aw.lineno)
         aw.value.walkabout(self)
@@ -1147,7 +1149,7 @@
             self.emit_op_arg(outer_op, elt_subitems)
         else:
             self.emit_op_arg(single_op, seen_star)
-    
+
     def _visit_assignment(self, node, elts, ctx):
         elt_count = len(elts) if elts else 0
         if ctx == ast.Store:
@@ -1210,14 +1212,8 @@
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
-                    # TODO: key.walkabout has to be called before d.values.walkabout
-                    # that would fix the error "keywords must be strings"
-                    # for some reason the keys and values seem to be in reverse order
-                    # in some cases, so another error has to be fixed in order for
-                    # this to work, otherwise it breaks everything
-                    # after fix: remove dirty fixes in pyopcode
+                    key.walkabout(self)
                     d.values[i].walkabout(self)
-                    key.walkabout(self)
                     elements += 1
         if elements or containers == 0:
             self.emit_op_arg(ops.BUILD_MAP, elements)
@@ -1230,7 +1226,7 @@
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
             is_unpacking = False
-    
+
     def visit_Set(self, s):
         self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
 
@@ -1286,7 +1282,7 @@
                     # If we ended up with more than one stararg, we need
                     # to concatenate them into a single sequence.
                     self.emit_op_arg(ops.BUILD_LIST_UNPACK, nsubargs)
-        
+
         # Repeat procedure for keyword args
         nseen = 0 # the number of keyword arguments on the stack following
         if keywords is not None:
@@ -1301,8 +1297,8 @@
                     nsubkwargs += 1
                 elif nsubkwargs:
                     # A keyword argument and we already have a dict.
+                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     kw.value.walkabout(self)
-                    self.load_const(self.space.wrap(kw.arg.decode('utf-8')))
                     nseen += 1
                 else:
                     # keyword argument
@@ -1340,7 +1336,7 @@

From pypy.commits at gmail.com  Mon Nov 14 19:19:56 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 Nov 2016 16:19:56 -0800 (PST)
Subject: [pypy-commit] pypy default: Move 2 scripts used only by PyPy to
 pypy/tool/
Message-ID: <582a54ac.45f6c20a.47cb3.af84@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88375:59c79103d5b0
Date: 2016-11-15 00:19 +0000
http://bitbucket.org/pypy/pypy/changeset/59c79103d5b0/

Log:	Move 2 scripts used only by PyPy to pypy/tool/

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -17,7 +17,7 @@
 from pypy.conftest import option as pypy_option
 
 from pypy.tool.pytest import appsupport
-from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir
+from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir
 from rpython.config.parse import parse_info
 
 pytest_plugins = "resultlog",
@@ -561,7 +561,7 @@
             watchdog_name = 'watchdog_nt.py'
         else:
             watchdog_name = 'watchdog.py'
-        watchdog_script = rpythondir.join('tool', watchdog_name)
+        watchdog_script = pypydir.join('tool', watchdog_name)
 
         regr_script = pypydir.join('tool', 'pytest',
                                    'run-script', 'regrverbose.py')
diff --git a/rpython/tool/watchdog.py b/pypy/tool/watchdog.py
rename from rpython/tool/watchdog.py
rename to pypy/tool/watchdog.py
diff --git a/rpython/tool/watchdog_nt.py b/pypy/tool/watchdog_nt.py
rename from rpython/tool/watchdog_nt.py
rename to pypy/tool/watchdog_nt.py

From pypy.commits at gmail.com  Tue Nov 15 01:08:10 2016
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 Nov 2016 22:08:10 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-injection: add r ops, test a bit more, fix,
 fix translation
Message-ID: <582aa64a.41a3c20a.927ec.a98f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-injection
Changeset: r88376:22a28b31d815
Date: 2016-11-15 08:07 +0200
http://bitbucket.org/pypy/pypy/changeset/22a28b31d815/

Log:	add r ops, test a bit more, fix, fix translation

diff --git a/pypy/module/cpyext/injection/numpy.py b/pypy/module/cpyext/injection/numpy.py
--- a/pypy/module/cpyext/injection/numpy.py
+++ b/pypy/module/cpyext/injection/numpy.py
@@ -66,6 +66,8 @@
     data = rffi.cast(rffi.DOUBLEP, py_obj.data)
     return W_Float64Object(data[index])
 
+internal_arr1 = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw', immortal=True)
+internal_arr2 = lltype.malloc(rffi.DOUBLEP.TO, 1, flavor='raw', immortal=True)
 def injected_op(space, w_self, w_other, op):
     # translate array_op in multiarray.c from c to rpython
     if isinstance(w_self, W_ArrayObject):
@@ -73,16 +75,16 @@
         data1 = rffi.cast(rffi.DOUBLEP, arr1.data)
         n1 = arr1.dimensions[0]
     else:
-        # XXX this should be a pointer to a float, not a list
-        data1 = [space.float_w(w_self),]
+        data1 = internal_arr1
+        data1[0] = space.float_w(w_self)
         n1 = 1
     if isinstance(w_other, W_ArrayObject):
         arr2 = rffi.cast(PyArrayObject, w_other.pyobj)
         data2 = rffi.cast(rffi.DOUBLEP, arr2.data)
         n2 = arr2.dimensions[0]
     else:
-        # XXX this should be a pointer to a float, not a list
-        data2 = [space.float_w(w_other),]
+        data2 = internal_arr2
+        data2[0] = space.float_w(w_other)
         n2 = 1
     if not (n1 == n2 or n1 == 1 or n2 == 1):
         raise oefmt(space.w_ValueError, 'dimension mismatch')
@@ -93,8 +95,8 @@
     r = rffi.cast(rffi.DOUBLEP, ret.pyobj.data)
     i1 = 0; i2 = 0
     for j in range(m):
-        if i1 > n1: i1 = 0
-        if i2 > n2: i2 = 0
+        if i1 >= n1: i1 = 0
+        if i2 >= n2: i2 = 0
         if op == 'mul':
             r[j] = data1[i1] * data2[i2]
         elif op == 'add':
@@ -127,13 +129,17 @@
     '__add__': interp2app(injected_add),
     '__sub__': interp2app(injected_sub),
     '__div__': interp2app(injected_div),
+    '__rmul__': interp2app(injected_mul),
+    '__radd__': interp2app(injected_add),
+    '__rsub__': interp2app(injected_sub),
+    '__rdiv__': interp2app(injected_div),
 }
 
 def inject_operator(space, name, dict_w, pto):
     assert name == 'numpy.ndarray'
     org = space.fromcache(Original)
     org.w_original_getitem = dict_w['__getitem__']
-    org.w_original_getitem = dict_w['__mul__']
+    org.w_original_mul = dict_w['__mul__']
     for key, w_value in org.injected_methods_w:
         dict_w[key] = w_value
     return W_ArrayObject.typedef
diff --git a/pypy/module/cpyext/injection/test/multiarray.c b/pypy/module/cpyext/injection/test/multiarray.c
--- a/pypy/module/cpyext/injection/test/multiarray.c
+++ b/pypy/module/cpyext/injection/test/multiarray.c
@@ -295,6 +295,7 @@
         if (PyErr_Occurred())
             return NULL;
         v1 = &tmp1;
+        n1 = 1;
     }
     if (obj2->ob_type == &PyArray_Type)
     {
@@ -307,8 +308,9 @@
         if (PyErr_Occurred())
             return NULL;
         v2 = &tmp2;
+        n2 = 1;
     }
-    if (!(n1 == n2 || n1 == 1 || n2 == 1))
+    if ( !(n1 == n2 || n1 == 1 || n2 == 1))
     {
         PyErr_SetString(PyExc_ValueError, "dimension mismatch");
         return NULL;
@@ -326,19 +328,19 @@
         switch (op)
         {
             case MULT:
-                r[j] = v1[i1] * v2[i2];
+                r[j] = v1[i1] * v2[i2] + 3;
                 break;
             case ADD:
-                r[j] = v1[i1] + v2[i2];
+                r[j] = v1[i1] + v2[i2] + 3;
                 break;
             case SUB:
-                r[j] = v1[i1] - v2[i2];
+                r[j] = v1[i1] - v2[i2] + 3;
                 break;
             case DIV:
                 if (v2[i2] == 0)
                     r[j] = Py_NAN;
                 else
-                    r[j] = v1[i1] / v2[i2];
+                    r[j] = v1[i1] / v2[i2] + 3;
                 break;
         }
     } 
diff --git a/pypy/module/cpyext/injection/test/test_numpy.py b/pypy/module/cpyext/injection/test/test_numpy.py
--- a/pypy/module/cpyext/injection/test/test_numpy.py
+++ b/pypy/module/cpyext/injection/test/test_numpy.py
@@ -44,7 +44,7 @@
         for i in range(100):
             a[i] = i
         b = a * a
-        assert b[10] == 100.0 + 42.0
+        assert b[10] == 100.0 + 3.0 + 42.0
 
     def test_injected_op(self):
         np = self.import_module(name='numpy.core.multiarray',
@@ -54,3 +54,8 @@
             a[i] = i
         b = a * a
         assert b[10] == 100.0
+        b = a * 2
+        assert b[5] == 10.0
+        b = 2 * a
+        assert b[5] == 10.0
+        

From pypy.commits at gmail.com  Tue Nov 15 02:15:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Nov 2016 23:15:22 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: like CPython, in 'log(int)',
 first try to cast the int to double, and
Message-ID: <582ab60a.113cc20a.214c3.09fc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88377:a95a63f9b6b3
Date: 2016-11-15 08:14 +0100
http://bitbucket.org/pypy/pypy/changeset/a95a63f9b6b3/

Log:	like CPython, in 'log(int)', first try to cast the int to double,
	and only if that overflows use the special very-long-int algorithm.

diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -189,12 +189,15 @@
 def _log_any(space, w_x, base):
     # base is supposed to be positive or 0.0, which means we use e
     try:
-        if space.isinstance_w(w_x, space.w_int):
+        try:
+            x = _get_double(space, w_x)
+        except OverflowError:
+            if not space.isinstance_w(w_x, space.w_int):
+                raise
             # special case to support log(extremely-large-long)
             num = space.bigint_w(w_x)
             result = num.log(base)
         else:
-            x = _get_double(space, w_x)
             if base == 10.0:
                 result = math.log10(x)
             elif base == 2.0:

From pypy.commits at gmail.com  Tue Nov 15 02:17:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Nov 2016 23:17:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: py3 fix
Message-ID: <582ab674.8a29c20a.26431.c868@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88378:a6e7189349da
Date: 2016-11-15 08:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a6e7189349da/

Log:	py3 fix

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -326,7 +326,7 @@
                     return -1
                 return a-b
             #
-            total = sys.maxint - 2147483647
+            total = sys.maxsize - 2147483647
             for i in range(100000):
                 total += f(i, 5)
             #

From pypy.commits at gmail.com  Tue Nov 15 02:54:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Nov 2016 23:54:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: remove xrange() test
Message-ID: <582abf18.581d1c0a.580fd.ca05@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88379:e6817784253f
Date: 2016-11-15 08:22 +0100
http://bitbucket.org/pypy/pypy/changeset/e6817784253f/

Log:	remove xrange() test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -127,38 +127,6 @@
             jump(..., descr=...)
             """)
 
-    def test_xrange_iter(self):
-        def main(n):
-            def g(n):
-                return xrange(n)
-            s = 0
-            for i in xrange(n):  # ID: for
-                tmp = g(n)
-                s += tmp[i]     # ID: getitem
-                a = 0
-            return s
-        #
-        log = self.run(main, [1000])
-        assert log.result == 1000 * 999 / 2
-        loop, = log.loops_by_filename(self.filepath)
-        assert loop.match("""
-        i15 = int_lt(i10, i11)
-        guard_true(i15, descr=...)
-        i17 = int_add(i10, 1)
-        setfield_gc(p9, i17, descr=<.* .*W_XRangeIterator.inst_current .*>)
-        guard_not_invalidated(descr=...)
-        i18 = force_token()
-        i84 = int_sub(i14, 1)
-        i21 = int_lt(i10, 0)
-        guard_false(i21, descr=...)
-        i22 = int_lt(i10, i14)
-        guard_true(i22, descr=...)
-        i23 = int_add_ovf(i6, i10)
-        guard_no_overflow(descr=...)
-        --TICK--
-        jump(..., descr=...)
-        """)
-
     def test_range_iter_simple(self):
         def main(n):
             def g(n):

From pypy.commits at gmail.com  Tue Nov 15 02:54:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 Nov 2016 23:54:02 -0800 (PST)
Subject: [pypy-commit] pypy default: Don't create log files that have the
 executable bit set
Message-ID: <582abf1a.e576c20a.f87aa.da28@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88380:0e2797574d3d
Date: 2016-11-15 08:53 +0100
http://bitbucket.org/pypy/pypy/changeset/0e2797574d3d/

Log:	Don't create log files that have the executable bit set

diff --git a/rpython/rlib/rjitlog/src/rjitlog.c b/rpython/rlib/rjitlog/src/rjitlog.c
--- a/rpython/rlib/rjitlog/src/rjitlog.c
+++ b/rpython/rlib/rjitlog/src/rjitlog.c
@@ -45,11 +45,11 @@
     filename = getenv("JITLOG");
 
     if (filename && filename[0]) {
-        // mode is 775
+        // mode is 644
 #ifdef _WIN32
-        int mode = _S_IWRITE | _S_IREAD | _S_IEXEC;
+        int mode = _S_IWRITE | _S_IREAD;
 #else        
-        mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH;
+        mode_t mode = 0644;
 #endif
         jitlog_fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
         if (jitlog_fd == -1) {

From pypy.commits at gmail.com  Tue Nov 15 03:49:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 00:49:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fight a lot but come to a conclusion:
 this extra int_lt() makes a bit of sense after all
Message-ID: <582acc0b.0209c20a.8f773.f63f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88381:db0d9c895763
Date: 2016-11-15 09:48 +0100
http://bitbucket.org/pypy/pypy/changeset/db0d9c895763/

Log:	Fight a lot but come to a conclusion: this extra int_lt() makes a
	bit of sense after all

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -143,13 +143,21 @@
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
             guard_not_invalidated?
-            i16 = int_ge(i11, i12)
-            guard_false(i16, descr=...)
+            # W_IntRangeStepOneIterator.next()
+            i16 = int_lt(i11, i12)
+            guard_true(i16, descr=...)
             i20 = int_add(i11, 1)
-            setfield_gc(p4, i20, descr=<.* .*W_AbstractSeqIterObject.inst_index .*>)
+            setfield_gc(p4, i20, descr=<.* .*W_IntRangeIterator.inst_current .*>)
             guard_not_invalidated?
             i21 = force_token()
             i88 = int_sub(i9, 1)
+
+            # Compared with pypy2, we get these two operations extra.
+            # I think the reason is that W_IntRangeStepOneIterator is used
+            # for any 'start' value, which might be negative.
+            i89 = int_lt(i11, 0)
+            guard_false(i89, descr=...)
+
             i25 = int_ge(i11, i9)
             guard_false(i25, descr=...)
             i27 = int_add_ovf(i7, i11)

From pypy.commits at gmail.com  Tue Nov 15 03:54:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 00:54:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test_range_iter_normal
Message-ID: <582acd5b.0a4cc20a.3a027.f954@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88382:33f878c5137c
Date: 2016-11-15 09:54 +0100
http://bitbucket.org/pypy/pypy/changeset/33f878c5137c/

Log:	test_range_iter_normal

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -141,7 +141,9 @@
         log = self.run(main, [1000])
         assert log.result == 1000 * 999 / 2
         loop, = log.loops_by_filename(self.filepath)
-        assert loop.match("""
+        assert loop.match(self.RANGE_ITER_STEP_1)
+
+    RANGE_ITER_STEP_1 = """
             guard_not_invalidated?
             # W_IntRangeStepOneIterator.next()
             i16 = int_lt(i11, i12)
@@ -164,9 +166,12 @@
             guard_no_overflow(descr=...)
             --TICK--
             jump(..., descr=...)
-        """)
+        """
 
     def test_range_iter_normal(self):
+        # Difference: range(n) => range(1, n).
+        # This difference doesn't really change anything in pypy3.5,
+        # but it used to in pypy2.7.
         def main(n):
             def g(n):
                 return range(n)
@@ -180,26 +185,7 @@
         log = self.run(main, [1000])
         assert log.result == 1000 * 999 / 2
         loop, = log.loops_by_filename(self.filepath)
-        assert loop.match("""
-            guard_not_invalidated?
-            i16 = int_ge(i11, i12)
-            guard_false(i16, descr=...)
-            i17 = int_mul(i11, i14)
-            i18 = int_add(i15, i17)
-            i20 = int_add(i11, 1)
-            setfield_gc(p4, i20, descr=<.* .*W_AbstractSeqIterObject.inst_index .*>)
-            guard_not_invalidated?
-            i21 = force_token()
-            i95 = int_sub(i9, 1)
-            i23 = int_lt(i18, 0)
-            guard_false(i23, descr=...)
-            i25 = int_ge(i18, i9)
-            guard_false(i25, descr=...)
-            i27 = int_add_ovf(i7, i18)
-            guard_no_overflow(descr=...)
-            --TICK--
-            jump(..., descr=...)
-        """)
+        assert loop.match(self.RANGE_ITER_STEP_1)
 
     def test_chain_of_guards(self):
         src = """

From pypy.commits at gmail.com  Tue Nov 15 04:07:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 01:07:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Move the loop out of space.wrap()
Message-ID: <582ad048.8b9a1c0a.f0db4.e651@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88383:c7dfd649a692
Date: 2016-11-15 10:06 +0100
http://bitbucket.org/pypy/pypy/changeset/c7dfd649a692/

Log:	Move the loop out of space.wrap()

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -155,18 +155,7 @@
             try:
                 unicode_x = x.decode('ascii')
             except UnicodeDecodeError:
-                # poor man's x.decode('ascii', 'replace'), since it's not
-                # supported by RPython
-                if not we_are_translated():
-                    print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
-                lst = []
-                for ch in x:
-                    ch = ord(ch)
-                    if ch > 127:
-                        lst.append(u'\ufffd')
-                    else:
-                        lst.append(unichr(ch))
-                unicode_x = u''.join(lst)
+                unicode_x = self._wrap_ascii_replace(x)
             return self.newunicode(unicode_x)
         if isinstance(x, unicode):
             return self.newunicode(x)
@@ -191,6 +180,22 @@
                 return W_LongObject.fromrarith_int(x)
         return self._wrap_not_rpython(x)
 
+    def _wrap_ascii_replace(self, x):
+        # XXX should disappear soon?
+        # poor man's x.decode('ascii', 'replace'), since it's not
+        # supported by RPython
+        if not we_are_translated():
+            print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x
+        lst = []
+        for ch in x:
+            ch = ord(ch)
+            if ch > 127:
+                lst.append(u'\ufffd')
+            else:
+                lst.append(unichr(ch))
+        unicode_x = u''.join(lst)
+        return unicode_x
+
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
         # _____ this code is here to support testing only _____

From pypy.commits at gmail.com  Tue Nov 15 04:14:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 01:14:30 -0800 (PST)
Subject: [pypy-commit] pypy default: Import cffi 1.9.1 (no difference from
 1.9.0)
Message-ID: <582ad1f6.272cc20a.38784.c826@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88384:a98b2c149e2d
Date: 2016-11-15 10:13 +0100
http://bitbucket.org/pypy/pypy/changeset/a98b2c149e2d/

Log:	Import cffi 1.9.1 (no difference from 1.9.0)

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.9.0
+Version: 1.9.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.9.0"
-__version_info__ = (1, 9, 0)
+__version__ = "1.9.1"
+__version_info__ = (1, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.9.0"
+                               "\ncompiled with cffi version: 1.9.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi, entrypoint
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.9.0"
+VERSION = "1.9.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.9.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.9.1", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Tue Nov 15 04:42:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 01:42:57 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Emphasis on "development version
 *of NumPy*". It's too easy to misread
Message-ID: <582ad8a1.05371c0a.7053b.f4fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r821:4a9ba4dfb1d7
Date: 2016-11-15 10:42 +0100
http://bitbucket.org/pypy/pypy.org/changeset/4a9ba4dfb1d7/

Log:	Emphasis on "development version *of NumPy*". It's too easy to
	misread "of PyPy".

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -115,7 +115,7 @@
 </ul>
 </div>
 <div class="section" id="python2-7-compatible-pypy-5-6-0">
-<h1>Python2.7 compatible PyPy 5.6.0</h1>
+<span id="release"></span><h1>Python2.7 compatible PyPy 5.6.0</h1>
 <ul class="simple">
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
@@ -228,8 +228,10 @@
 the Python side and NumPy objects are mediated through the slower cpyext
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).</p>
-<p>Installation works using the current developement version of NumPy, since
-PyPy specific changes have been merged but not in a release version.
+<p>Installation works on any recent PyPy (the <a class="reference internal" href="#release">release</a> above is fine),
+but you need the current developement version <em>of NumPy</em>.  The reason
+is that some PyPy-specific fixes have been merged back into NumPy,
+and they are not yet in a released version of NumPy.
 For example, without using a virtualenv:</p>
 <pre class="literal-block">
 $ ./pypy-xxx/bin/pypy -m ensurepip
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -76,6 +76,8 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
+.. _release:
+
 Python2.7 compatible PyPy 5.6.0
 -------------------------------
 
@@ -251,8 +253,10 @@
 layer (which hurts a few benchmarks that do a lot of element-by-element
 array accesses, for example).
 
-Installation works using the current developement version of NumPy, since
-PyPy specific changes have been merged but not in a release version.
+Installation works on any recent PyPy (the release_ above is fine),
+but you need the current developement version *of NumPy*.  The reason
+is that some PyPy-specific fixes have been merged back into NumPy,
+and they are not yet in a released version of NumPy.
 For example, without using a virtualenv::
 
     $ ./pypy-xxx/bin/pypy -m ensurepip

From pypy.commits at gmail.com  Tue Nov 15 04:45:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 01:45:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test_clear_locals() from cbba0303793c
Message-ID: <582ad954.83f8c20a.c5d11.0d38@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88385:6299e1a8556b
Date: 2016-11-15 10:43 +0100
http://bitbucket.org/pypy/pypy/changeset/6299e1a8556b/

Log:	Fix test_clear_locals() from cbba0303793c

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -962,7 +962,12 @@
 
         # clear the locals, including the cell/free vars, and the stack
         for i in range(len(self.locals_cells_stack_w)):
-            self.locals_cells_stack_w[i] = None
+            w_oldvalue = self.locals_cells_stack_w[i]
+            if isinstance(w_oldvalue, Cell):
+                w_newvalue = Cell()
+            else:
+                w_newvalue = None
+            self.locals_cells_stack_w[i] = w_newvalue
         self.valuestackdepth = 0
 
     def _convert_unexpected_exception(self, e):

From pypy.commits at gmail.com  Tue Nov 15 08:50:19 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 15 Nov 2016 05:50:19 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: three more in _socket
Message-ID: <582b129b.8b9a1c0a.f0db4.5607@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88386:27a8f3f8230c
Date: 2016-11-11 18:17 +0100
http://bitbucket.org/pypy/pypy/changeset/27a8f3f8230c/

Log:	three more in _socket

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -24,8 +24,8 @@
     elif isinstance(addr, rsocket.INET6Address):
         return space.newtuple([space.newtext(addr.get_host()),
                                space.newint(addr.get_port()),
-                               space.wrap(addr.get_flowinfo()), # YYY
-                               space.wrap(addr.get_scope_id())]) # YYY
+                               space.newint(addr.get_flowinfo()),
+                               space.newint(addr.get_scope_id())])
     elif rsocket.HAS_AF_PACKET and isinstance(addr, rsocket.PacketAddress):
         return space.newtuple([space.newtext(addr.get_ifname(fd)),
                                space.newint(addr.get_protocol()),
@@ -556,7 +556,7 @@
                 if value_ptr:
                     lltype.free(value_ptr, flavor='raw')
 
-            return space.wrap(recv_ptr[0]) # YYY
+            return space.newint(recv_ptr[0])
         finally:
             lltype.free(recv_ptr, flavor='raw')
 

From pypy.commits at gmail.com  Tue Nov 15 08:50:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 15 Nov 2016 05:50:21 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix two typos in the array module
Message-ID: <582b129d.69efc20a.c9705.7043@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88387:0292533aef1d
Date: 2016-11-15 14:49 +0100
http://bitbucket.org/pypy/pypy/changeset/0292533aef1d/

Log:	fix two typos in the array module

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -57,7 +57,7 @@
 
 
 def descr_typecode(space, self):
-    return space.newint(self.typecode)
+    return space.newtext(self.typecode)
 
 arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
                               reds='auto')
@@ -272,7 +272,7 @@
             elems = max(0, len(item) - (len(item) % self.itemsize))
             if n != 0:
                 item = item[0:elems]
-            self.descr_fromstring(space, space.newint(item))
+            self.descr_fromstring(space, space.newbytes(item))
             raise oefmt(space.w_EOFError, "not enough items in file")
         self.descr_fromstring(space, w_item)
 

From pypy.commits at gmail.com  Tue Nov 15 10:00:07 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 15 Nov 2016 07:00:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: copy Lib/ssl.py from cpython. Pass
 lots of more tests in test_ssl.py
Message-ID: <582b22f7.c1341c0a.d3e82.772d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88388:cdbfa886bf3b
Date: 2016-11-15 15:59 +0100
http://bitbucket.org/pypy/pypy/changeset/cdbfa886bf3b/

Log:	copy Lib/ssl.py from cpython. Pass lots of more tests in test_ssl.py

diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -145,6 +145,7 @@
 from socket import SOL_SOCKET, SO_TYPE
 import base64        # for DER-to-PEM translation
 import errno
+import warnings
 
 
 socket_error = OSError  # keep that public name in module namespace
@@ -405,12 +406,16 @@
 
     def _load_windows_store_certs(self, storename, purpose):
         certs = bytearray()
-        for cert, encoding, trust in enum_certificates(storename):
-            # CA certs are never PKCS#7 encoded
-            if encoding == "x509_asn":
-                if trust is True or purpose.oid in trust:
-                    certs.extend(cert)
-        self.load_verify_locations(cadata=certs)
+        try:
+            for cert, encoding, trust in enum_certificates(storename):
+                # CA certs are never PKCS#7 encoded
+                if encoding == "x509_asn":
+                    if trust is True or purpose.oid in trust:
+                        certs.extend(cert)
+        except PermissionError:
+            warnings.warn("unable to enumerate Windows certificate store")
+        if certs:
+            self.load_verify_locations(cadata=certs)
         return certs
 
     def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
@@ -560,7 +565,7 @@
         server hostame is set."""
         return self._sslobj.server_hostname
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to 'len' bytes from the SSL object and return them.
 
         If 'buffer' is provided, read into this buffer and return the number of
@@ -569,7 +574,7 @@
         if buffer is not None:
             v = self._sslobj.read(len, buffer)
         else:
-            v = self._sslobj.read(len or 1024)
+            v = self._sslobj.read(len)
         return v
 
     def write(self, data):
@@ -745,7 +750,8 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-            except:
+
+            except (OSError, ValueError):
                 self.close()
                 raise
 
@@ -774,7 +780,7 @@
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
 
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -2889,8 +2889,10 @@
                 # will be full and the call will block
                 buf = bytearray(8192)
                 def fill_buffer():
+                    i = 0
                     while True:
                         s.send(buf)
+                        i += 1
                 self.assertRaises((ssl.SSLWantWriteError,
                                    ssl.SSLWantReadError), fill_buffer)
 
diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -25,6 +25,9 @@
 static const long Cryptography_HAS_GET_SERVER_TMP_KEY;
 static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE;
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS;
+static const long Cryptography_HAS_NPN_NEGOTIATED;
+
+static const long Cryptography_OPENSSL_NPN_NEGOTIATED;
 
 /* Internally invented symbol to tell us if SNI is supported */
 static const long Cryptography_HAS_TLSEXT_HOSTNAME;
@@ -435,6 +438,7 @@
 long SSL_CTX_sess_misses(SSL_CTX *);
 long SSL_CTX_sess_timeouts(SSL_CTX *);
 long SSL_CTX_sess_cache_full(SSL_CTX *);
+
 """
 
 CUSTOMIZATIONS = """
@@ -689,6 +693,14 @@
 
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1;
 
+#ifdef OPENSSL_NPN_NEGOTIATED
+static const long Cryptography_OPENSSL_NPN_NEGOTIATED = OPENSSL_NPN_NEGOTIATED;
+static const long Cryptography_HAS_NPN_NEGOTIATED = 1;
+#else
+static const long Cryptography_OPENSSL_NPN_NEGOTIATED = 0;
+static const long Cryptography_HAS_NPN_NEGOTIATED = 0;
+#endif
+
 /* in OpenSSL 1.1.0 the SSL_ST values were renamed to TLS_ST and several were
    removed */
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 || defined(LIBRESSL_VERSION_NUMBER)
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -7,8 +7,9 @@
 from _openssl import lib
 from openssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
-from openssl._stdssl.utility import _str_with_len, _bytes_with_len, _str_to_ffi_buffer
-from openssl._stdssl.error import (ssl_error, ssl_lib_error, ssl_socket_error,
+from openssl._stdssl.utility import (_str_with_len, _bytes_with_len,
+    _str_to_ffi_buffer, _str_from_buf)
+from openssl._stdssl.error import (ssl_error, pyssl_error,
         SSLError, SSLZeroReturnError, SSLWantReadError,
         SSLWantWriteError, SSLSyscallError,
         SSLEOFError)
@@ -35,7 +36,7 @@
 HAS_ECDH = bool(lib.Cryptography_HAS_ECDH)
 HAS_SNI = bool(lib.Cryptography_HAS_TLSEXT_HOSTNAME)
 HAS_ALPN = bool(lib.Cryptography_HAS_ALPN)
-HAS_NPN = False
+HAS_NPN = lib.Cryptography_HAS_NPN_NEGOTIATED
 HAS_TLS_UNIQUE = True
 
 CLIENT = 0
@@ -61,6 +62,8 @@
 SSL_CLIENT = 0
 SSL_SERVER = 1
 
+SSL_CB_MAXLEN=128
+
 if lib.Cryptography_HAS_SSL2:
     PROTOCOL_SSLv2  = 0
 PROTOCOL_SSLv3  = 1
@@ -73,7 +76,7 @@
 
 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
 
-from enum import Enum as _Enum, IntEnum as _IntEnum
+from enum import IntEnum as _IntEnum
 _IntEnum._convert('_SSLMethod', __name__,
         lambda name: name.startswith('PROTOCOL_'))
 
@@ -88,6 +91,14 @@
 # TODO threads?
 lib.OpenSSL_add_all_algorithms()
 
+def _socket_timeout(s):
+    if s is None:
+        return 0.0
+    t = s.gettimeout()
+    if t is None:
+        return -1.0
+    return t
+
 class PasswordInfo(object):
     callable = None
     password = None
@@ -139,7 +150,7 @@
     if sock is None or timeout == 0:
         return SOCKET_IS_NONBLOCKING
     elif timeout < 0:
-        t = sock.gettimeout() or 0
+        t = _socket_timeout(sock)
         if t > 0:
             return SOCKET_HAS_TIMED_OUT
         else:
@@ -219,7 +230,7 @@
         # If the socket is in non-blocking mode or timeout mode, set the BIO
         # to non-blocking mode (blocking is the default)
         #
-        timeout = sock.gettimeout() or 0
+        timeout = _socket_timeout(sock)
         if sock and timeout >= 0:
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), 1)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), 1)
@@ -246,6 +257,8 @@
         self.owner = None
         self.server_hostname = None
         self.socket = None
+        self.alpn_protocols = ffi.NULL
+        self.npn_protocols = ffi.NULL
 
     @property
     def context(self):
@@ -260,15 +273,13 @@
         if sock is None:
             raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
         ssl = self.ssl
-        timeout = 0
+        timeout = _socket_timeout(sock)
         if sock:
-            timeout = sock.gettimeout() or 0
             nonblocking = timeout >= 0
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
         has_timeout = timeout > 0
-        has_timeout = (timeout > 0);
         deadline = -1
         if has_timeout:
             # REVIEW, cpython uses a monotonic clock here
@@ -296,7 +307,7 @@
                 sockstate = SOCKET_OPERATION_OK
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise SSLError("The handshake operation timed out")
+                raise socket.timeout("The handshake operation timed out")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise SSLError("Underlying socket has been closed.")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
@@ -306,7 +317,7 @@
             if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
                 break
         if ret < 1:
-            raise ssl_lib_error()
+            raise pyssl_error(self, ret)
 
         if self.peer_cert != ffi.NULL:
             lib.X509_free(self.peer_cert)
@@ -335,15 +346,19 @@
     def write(self, bytestring):
         deadline = 0
         b = _str_to_ffi_buffer(bytestring)
-        sock = self.get_socket_or_None()
+        sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
+
+        if len(b) > sys.maxsize:
+            raise OverflowError("string longer than %d bytes" % sys.maxsize)
+
+        timeout = _socket_timeout(sock)
         if sock:
-            timeout = sock.gettimeout() or 0
             nonblocking = timeout >= 0
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
-        timeout = sock.gettimeout() or 0
+
         has_timeout = timeout > 0
         if has_timeout:
             # TODO monotonic clock?
@@ -351,7 +366,7 @@
 
         sockstate = _ssl_select(sock, 1, timeout)
         if sockstate == SOCKET_HAS_TIMED_OUT:
-            raise socket.TimeoutError("The write operation timed out")
+            raise socket.timeout("The write operation timed out")
         elif sockstate == SOCKET_HAS_BEEN_CLOSED:
             raise ssl_error("Underlying socket has been closed.")
         elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
@@ -378,7 +393,7 @@
                 sockstate = SOCKET_OPERATION_OK
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise socket.TimeoutError("The write operation timed out")
+                raise socket.timeout("The write operation timed out")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise ssl_error("Underlying socket has been closed.")
             elif sockstate == SOCKET_IS_NONBLOCKING:
@@ -389,13 +404,15 @@
         if length > 0:
             return length
         else:
-            raise ssl_lib_error()
-            # return PySSL_SetError(self, len, __FILE__, __LINE__);
+            raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
         sock = self.get_socket_or_None()
         ssl = self.ssl
 
+        if length < 0 and buffer_into is None:
+            raise ValueError("size should not be negative")
+
         if sock is None:
             raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
 
@@ -403,20 +420,20 @@
             dest = _buffer_new(length)
             mem = dest
         else:
-            import pdb; pdb.set_trace()
             mem = ffi.from_buffer(buffer_into)
             if length <= 0 or length > len(buffer_into):
-                if len(buffer_into) != length:
+                length = len(buffer_into)
+                if length > sys.maxsize:
                     raise OverflowError("maximum length can't fit in a C 'int'")
 
         if sock:
-            timeout = sock.gettimeout() or 0
+            timeout = _socket_timeout(sock)
             nonblocking = timeout >= 0
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), nonblocking)
 
         deadline = 0
-        timeout = sock.gettimeout() or 0
+        timeout = _socket_timeout(sock)
         has_timeout = timeout > 0
         if has_timeout:
             # TODO monotonic clock?
@@ -448,28 +465,29 @@
                 sockstate = SOCKET_OPERATION_OK
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise socket.TimeoutError("The read operation timed out")
+                raise socket.timeout("The read operation timed out")
             elif sockstate == SOCKET_IS_NONBLOCKING:
                 break
             if not (err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE):
                 break
 
-        if count <= 0:
-            raise ssl_socket_error(self, err)
+        if count <= 0 and not shutdown:
+            raise pyssl_error(self, count)
 
         if not buffer_into:
             return _bytes_with_len(dest, count)
 
         return count
 
-    def selected_alpn_protocol(self):
-        out = ffi.new("const unsigned char **")
-        outlen = ffi.new("unsigned int*")
+    if HAS_ALPN:
+        def selected_alpn_protocol(self):
+            out = ffi.new("const unsigned char **")
+            outlen = ffi.new("unsigned int*")
 
-        lib.SSL_get0_alpn_selected(self.ssl, out, outlen);
-        if out == ffi.NULL:
-            return None
-        return _str_with_len(ffi.cast("char*",out[0]), outlen[0]);
+            lib.SSL_get0_alpn_selected(self.ssl, out, outlen);
+            if out[0] == ffi.NULL:
+                return None
+            return _str_with_len(out[0], outlen[0]);
 
     def shared_ciphers(self):
         sess = lib.SSL_get_session(self.ssl)
@@ -519,6 +537,11 @@
             return None
         return self.socket()
 
+    def get_socket_or_connection_gone(self):
+        if self.socket is None:
+            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+        return self.socket()
+
     def shutdown(self):
         sock = self.get_socket_or_None()
         nonblocking = False
@@ -529,7 +552,7 @@
             if sock.fileno() < 0:
                 raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
 
-            timeout = sock.gettimeout() or 0
+            timeout = _socket_timeout(sock)
             nonblocking = timeout >= 0
             if sock and timeout >= 0:
                 lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), nonblocking)
@@ -588,9 +611,9 @@
 
             if sockstate == SOCKET_HAS_TIMED_OUT:
                 if ssl_err == SSL_ERROR_WANT_READ:
-                    raise socket.TimeoutError("The read operation timed out")
+                    raise socket.timeout("The read operation timed out")
                 else:
-                    raise socket.TimeoutError("The write operation timed out")
+                    raise socket.timeout("The write operation timed out")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
                 raise ssl_error("Underlying socket too large for select().")
             elif sockstate != SOCKET_OPERATION_OK:
@@ -598,12 +621,46 @@
                 break;
 
         if err < 0:
-            raise ssl_socket_error(self, err)
+            raise pyssl_error(self, err)
         if sock:
             return sock
         else:
             return None
 
+    def pending(self):
+        # TODO PySSL_BEGIN_ALLOW_THREADS
+        count = lib.SSL_pending(self.ssl)
+        # TODO PySSL_END_ALLOW_THREADS
+        if count < 0:
+            raise pyssl_error(self, count)
+        else:
+            return count
+
+    def tls_unique_cb(self):
+        buf = ffi.new("char[%d]" % SSL_CB_MAXLEN)
+
+        if lib.SSL_session_reused(self.ssl) ^ (not self.socket_type):
+            # if session is resumed XOR we are the client
+            length = lib.SSL_get_finished(self.ssl, buf, SSL_CB_MAXLEN)
+        else:
+            # if a new session XOR we are the server
+            length = lib.SSL_get_peer_finished(self.ssl, buf, SSL_CB_MAXLEN)
+
+        # It cannot be negative in current OpenSSL version as of July 2011
+        if length == 0:
+            return None
+
+        return _bytes_with_len(buf, length)
+
+    if HAS_NPN:
+        def selected_npn_protocol(self):
+            out = ffi.new("unsigned char**")
+            outlen = ffi.new("unsigned int*")
+            lib.SSL_get0_next_proto_negotiated(self.ssl, out, outlen)
+            if (out[0] == ffi.NULL):
+                return None
+            return _str_with_len(out[0], outlen[0])
+
 
 def _fs_decode(name):
     # TODO return PyUnicode_DecodeFSDefault(short_name);
@@ -638,7 +695,8 @@
     SSL_CTX_STATS.append((name, getattr(lib, attr)))
 
 class _SSLContext(object):
-    __slots__ = ('ctx', '_check_hostname', 'servername_callback')
+    __slots__ = ('ctx', '_check_hostname', 'servername_callback',
+                 'alpn_protocols', 'npn_protocols')
 
     def __new__(cls, protocol):
         self = object.__new__(cls)
@@ -684,12 +742,8 @@
                 lib.SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
                 key = lib.EC_KEY_new_by_curve_name(lib.NID_X9_62_prime256v1)
-                if not key:
-                    raise ssl_lib_error()
-                try:
-                    lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
-                finally:
-                    lib.EC_KEY_free(key)
+                lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
+                lib.EC_KEY_free(key)
         if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
             store = lib.SSL_CTX_get_cert_store(self.ctx)
             lib.X509_STORE_set_flags(store, lib.X509_V_FLAG_TRUSTED_FIRST)
@@ -820,7 +874,7 @@
                     lib.ERR_clear_error()
                     raise OSError(_errno, "Error")
                 else:
-                    raise ssl_lib_error()
+                    raise ssl_error(None)
 
             ffi.errno = 0
             buf = _str_to_ffi_buffer(keyfile)
@@ -835,7 +889,7 @@
                     lib.ERR_clear_error()
                     raise OSError(_errno, None)
                 else:
-                    raise ssl_lib_error()
+                    raise ssl_error(None)
 
             ret = lib.SSL_CTX_check_private_key(self.ctx)
             if ret != 1:
@@ -890,7 +944,7 @@
                     lib.ERR_clear_error()
                     raise OSError(_errno, '')
                 else:
-                    raise ssl_lib_error()
+                    raise ssl_error(None)
 
     def _add_ca_certs(self, data, size, ca_file_type):
         biobuf = lib.BIO_new_mem_buf(data, size)
@@ -935,7 +989,7 @@
                 # EOF PEM file, not an error
                 lib.ERR_clear_error()
             else:
-                raise ssl_lib_error()
+                raise ssl_error(None)
         finally:
             lib.BIO_free(biobuf)
 
@@ -969,13 +1023,6 @@
 #        if ctx:
 #            self.ctx = lltype.nullptr(SSL_CTX.TO)
 #            libssl_SSL_CTX_free(ctx)
-#
-#    @staticmethod
-#    @unwrap_spec(protocol=int)
-#    def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_SSL23):
-#        self = space.allocate_instance(SSLContext, w_subtype)
-#        self.__init__(space, protocol)
-#        return space.wrap(self)
 
     def session_stats(self):
         stats = {}
@@ -987,34 +1034,6 @@
         if not lib.SSL_CTX_set_default_verify_paths(self.ctx):
             raise ssl_error("")
 
-#    def descr_get_options(self, space):
-#        return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
-#
-#    def descr_set_options(self, space, w_new_opts):
-#        new_opts = space.int_w(w_new_opts)
-#        opts = libssl_SSL_CTX_get_options(self.ctx)
-#        clear = opts & ~new_opts
-#        set = ~opts & new_opts
-#        if clear:
-#            if HAVE_SSL_CTX_CLEAR_OPTIONS:
-#                libssl_SSL_CTX_clear_options(self.ctx, clear)
-#            else:
-#                raise oefmt(space.w_ValueError,
-#                            "can't clear options before OpenSSL 0.9.8m")
-#        if set:
-#            libssl_SSL_CTX_set_options(self.ctx, set)
-#
-#    def descr_get_check_hostname(self, space):
-#        return space.newbool(self.check_hostname)
-#
-#    def descr_set_check_hostname(self, space, w_obj):
-#        check_hostname = space.is_true(w_obj)
-#        if check_hostname and libssl_SSL_CTX_get_verify_mode(self.ctx) == SSL_VERIFY_NONE:
-#            raise oefmt(space.w_ValueError,
-#                        "check_hostname needs a SSL context with either "
-#                        "CERT_OPTIONAL or CERT_REQUIRED")
-#        self.check_hostname = check_hostname
-#
     def load_dh_params(self, filepath):
         ffi.errno = 0
         if filepath is None:
@@ -1037,57 +1056,13 @@
                 lib.ERR_clear_error()
                 raise OSError(_errno, '')
             else:
-                raise ssl_lib_error()
+                raise ssl_error(None)
         try:
             if lib.SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
-                raise ssl_lib_error()
+                raise ssl_error(None)
         finally:
             lib.DH_free(dh)
 
-#    def cert_store_stats_w(self, space):
-#        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-#        x509 = 0
-#        x509_ca = 0
-#        crl = 0
-#        for i in range(libssl_sk_X509_OBJECT_num(store[0].c_objs)):
-#            obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i)
-#            if intmask(obj.c_type) == X509_LU_X509:
-#                x509 += 1
-#                if libssl_X509_check_ca(
-#                        libssl_pypy_X509_OBJECT_data_x509(obj)):
-#                    x509_ca += 1
-#            elif intmask(obj.c_type) == X509_LU_CRL:
-#                crl += 1
-#            else:
-#                # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
-#                # As far as I can tell they are internal states and never
-#                # stored in a cert store
-#                pass
-#        w_result = space.newdict()
-#        space.setitem(w_result,
-#                      space.wrap('x509'), space.wrap(x509))
-#        space.setitem(w_result,
-#                      space.wrap('x509_ca'), space.wrap(x509_ca))
-#        space.setitem(w_result,
-#                      space.wrap('crl'), space.wrap(crl))
-#        return w_result
-#
-#    @unwrap_spec(protos='bufferstr')
-#    def set_npn_protocols_w(self, space, protos):
-#        if not HAS_NPN:
-#            raise oefmt(space.w_NotImplementedError,
-#                        "The NPN extension requires OpenSSL 1.0.1 or later.")
-#
-#        self.npn_protocols = SSLNpnProtocols(self.ctx, protos)
-#
-#    @unwrap_spec(protos='bufferstr')
-#    def set_alpn_protocols_w(self, space, protos):
-#        if not HAS_ALPN:
-#            raise oefmt(space.w_NotImplementedError,
-#                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
-#
-#        self.alpn_protocols = SSLAlpnProtocols(self.ctx, protos)
-#
     def get_ca_certs(self, binary_form=None):
         binary_mode = bool(binary_form)
         _list = []
@@ -1117,7 +1092,7 @@
             raise ValueError("unknown elliptic curve name '%s'" % name)
         key = lib.EC_KEY_new_by_curve_name(nid)
         if not key:
-            raise ssl_lib_error()
+            raise ssl_error(None)
         try:
             lib.SSL_CTX_set_tmp_ecdh(self.ctx, key)
         finally:
@@ -1139,6 +1114,29 @@
         lib.Cryptography_SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
         lib.Cryptography_SSL_CTX_set_tlsext_servername_arg(self.ctx, ffi.new_handle(callback_struct))
 
+    def _set_alpn_protocols(self, protos):
+        if HAS_ALPN:
+            self.alpn_protocols = protocols = ffi.from_buffer(protos)
+            length = len(protocols)
+
+            if lib.SSL_CTX_set_alpn_protos(self.ctx,ffi.cast("unsigned char*", protocols), length):
+                return MemoryError()
+            handle = ffi.new_handle(self)
+            lib.SSL_CTX_set_alpn_select_cb(self.ctx, select_alpn_callback, handle)
+        else:
+            raise NotImplementedError("The ALPN extension requires OpenSSL 1.0.2 or later.")
+
+    def _set_npn_protocols(self, protos):
+        if HAS_NPN:
+            self.npn_protocols = ffi.from_buffer(protos)
+            handle = ffi.new_handle(self)
+            lib.SSL_CTX_set_next_protos_advertised_cb(self.ctx, advertise_npn_callback, handle)
+            lib.SSL_CTX_set_next_proto_select_cb(self.ctx, select_npn_callback, handle)
+        else:
+            raise NotImplementedError("The NPN extension requires OpenSSL 1.0.1 or later.")
+
+
+
 @ffi.callback("void(void)")
 def _servername_callback(ssl, ad, arg):
     struct = ffi.from_handle(arg)
@@ -1209,9 +1207,6 @@
     ctx = None
 SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
 
-def _str_from_buf(buf):
-    return ffi.string(buf).decode('utf-8')
-
 def _asn1obj2py(obj):
     nid = lib.OBJ_obj2nid(obj)
     if nid == lib.NID_undef:
@@ -1282,7 +1277,7 @@
             raise ssl_error("cannot write() after write_eof()")
         nbytes = lib.BIO_write(self.bio, buf, len(buf));
         if nbytes < 0:
-            raise ssl_lib_error()
+            raise ssl_error(None)
         return nbytes
 
     def write_eof(self):
@@ -1311,6 +1306,7 @@
     def pending(self):
         return lib.BIO_ctrl_pending(self.bio)
 
+
 RAND_status = lib.RAND_status
 RAND_add = lib.RAND_add
 
@@ -1348,7 +1344,7 @@
 #            target = PyBytes_FromString(tmp); } \
 #        if (!target) goto error; \
 #    }
-    # XXX
+    # REVIEW
     return ffi.string(buf).decode(sys.getfilesystemencoding())
 
 def get_default_verify_paths():
@@ -1367,3 +1363,50 @@
         return odir
 
     return (ofile_env, ofile, odir_env, odir);
+
+ at ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
+def select_alpn_callback(ssl, out, outlen, client_protocols, client_protocols_len, args):
+    ctx = ffi.from_handle(args)
+    return do_protocol_selection(1, out, outlen,
+                                 ffi.cast("unsigned char*",ctx.alpn_protocols), len(ctx.alpn_protocols),
+                                 client_protocols, client_protocols_len)
+
+ at ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
+def select_npn_callback(ssl, out, outlen, server_protocols, server_protocols_len, args):
+    ctx = ffi.from_handle(args)
+    return do_protocol_selection(0, out, outlen, server_protocols, server_protocols_len,
+                                 ffi.cast("unsigned char*",ctx.npn_protocols), len(ctx.npn_protocols))
+
+
+ at ffi.callback("int(SSL*,const unsigned char**, unsigned int*, void*)")
+def advertise_npn_callback(ssl, data, length, args):
+    ctx = ffi.from_handle(args)
+
+    if not ctx.npn_protocols:
+        data[0] = ffi.new("unsigned char*", b"")
+        length[0] = 0
+    else:
+        data[0] = ffi.cast("unsigned char*",ctx.npn_protocols)
+        length[0] = len(ctx.npn_protocols)
+
+    return lib.SSL_TLSEXT_ERR_OK
+
+
+if lib.Cryptography_HAS_NPN_NEGOTIATED:
+    def do_protocol_selection(alpn, out, outlen, server_protocols, server_protocols_len,
+                                                 client_protocols, client_protocols_len):
+        if client_protocols == ffi.NULL:
+            client_protocols = b""
+            client_protocols_len = 0
+        if server_protocols == ffi.NULL:
+            server_protocols = ""
+            server_protocols_len = 0
+
+        ret = lib.SSL_select_next_proto(out, outlen,
+                                        server_protocols, server_protocols_len,
+                                        client_protocols, client_protocols_len);
+        if alpn and ret != lib.Cryptography_OPENSSL_NPN_NEGOTIATED:
+            return lib.SSL_TLSEXT_ERR_NOACK
+
+        return lib.SSL_TLSEXT_ERR_OK
+
diff --git a/lib_pypy/openssl/_stdssl/certificate.py b/lib_pypy/openssl/_stdssl/certificate.py
--- a/lib_pypy/openssl/_stdssl/certificate.py
+++ b/lib_pypy/openssl/_stdssl/certificate.py
@@ -4,7 +4,7 @@
 from _openssl import ffi
 from _openssl import lib
 from openssl._stdssl.utility import _string_from_asn1, _str_with_len, _bytes_with_len
-from openssl._stdssl.error import ssl_error, ssl_socket_error, ssl_lib_error
+from openssl._stdssl.error import ssl_error, pyssl_error
 
 X509_NAME_MAXLEN = 256
 
@@ -12,13 +12,13 @@
     buf = ffi.new("char[]", X509_NAME_MAXLEN)
     length = lib.OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
     if length < 0:
-        raise ssl_socket_error(None, 0)
+        raise ssl_error(None)
     name = _str_with_len(buf, length)
 
     buf_ptr = ffi.new("unsigned char**")
     length = lib.ASN1_STRING_to_UTF8(buf_ptr, value)
     if length < 0:
-        raise ssl_socket_error(None, 0)
+        raise ssl_error(None)
     try:
         value = _str_with_len(buf_ptr[0], length)
     finally:
@@ -167,7 +167,7 @@
     length = lib.BIO_gets(biobuf, STATIC_BIO_BUF, len(STATIC_BIO_BUF)-1)
     if length < 0:
         if biobuf: lib.BIO_free(biobuf)
-        raise ssl_lib_error()
+        raise ssl_error(None)
     return _str_with_len(STATIC_BIO_BUF, length)
 
 def _decode_certificate(certificate):
@@ -198,7 +198,7 @@
         buf = ffi.new("char[]", 2048)
         length = lib.BIO_gets(biobuf, buf, len(buf)-1)
         if length < 0:
-            raise ssl_lib_error()
+            raise ssl_error(None)
         retval["serialNumber"] = _str_with_len(buf, length)
 
         lib.BIO_reset(biobuf);
@@ -206,7 +206,7 @@
         lib.ASN1_TIME_print(biobuf, notBefore);
         length = lib.BIO_gets(biobuf, buf, len(buf)-1);
         if length < 0:
-            raise ssl_lib_error()
+            raise ssl_error(None)
         retval["notBefore"] = _str_with_len(buf, length)
 
         lib.BIO_reset(biobuf);
@@ -214,7 +214,7 @@
         lib.ASN1_TIME_print(biobuf, notAfter);
         length = lib.BIO_gets(biobuf, buf, len(buf)-1);
         if length < 0:
-            raise ssl_lib_error()
+            raise ssl_error(None)
         retval["notAfter"] = _str_with_len(buf, length)
 
         # Now look for subjectAltName
@@ -332,7 +332,7 @@
     buf_ptr[0] = ffi.NULL
     length = lib.i2d_X509(certificate, buf_ptr)
     if length < 0:
-        raise ssl_lib_error()
+        raise ssl_error(None)
     try:
         return _bytes_with_len(ffi.cast("char*",buf_ptr[0]), length)
     finally:
diff --git a/lib_pypy/openssl/_stdssl/error.py b/lib_pypy/openssl/_stdssl/error.py
--- a/lib_pypy/openssl/_stdssl/error.py
+++ b/lib_pypy/openssl/_stdssl/error.py
@@ -1,7 +1,7 @@
 from _openssl import ffi
 from _openssl import lib
 
-from openssl._stdssl.utility import _string_from_asn1, _str_to_ffi_buffer
+from openssl._stdssl.utility import _string_from_asn1, _str_to_ffi_buffer, _str_from_buf
 
 SSL_ERROR_NONE = 0
 SSL_ERROR_SSL = 1
@@ -42,34 +42,13 @@
 class SSLEOFError(SSLError):
     """ SSL/TLS connection terminated abruptly. """
 
-def ssl_lib_error():
-    errcode = lib.ERR_peek_last_error()
-    lib.ERR_clear_error()
-    return ssl_error(None, 0, None, errcode)
-
-def ssl_error(msg, errno=0, errtype=None, errcode=0):
-    reason_str = None
-    lib_str = None
-    if errcode:
-        err_lib = lib.ERR_GET_LIB(errcode)
-        err_reason = lib.ERR_GET_REASON(errcode)
-        reason_str = ERR_CODES_TO_NAMES.get((err_lib, err_reason), None)
-        lib_str = LIB_CODES_TO_NAMES.get(err_lib, None)
-        msg = ffi.string(lib.ERR_reason_error_string(errcode)).decode('utf-8')
-    if not msg:
-        msg = "unknown error"
-    if reason_str and lib_str:
-        msg = "[%s: %s] %s" % (lib_str, reason_str, msg)
-    elif lib_str:
-        msg = "[%s] %s" % (lib_str, msg)
-
-    if errno or errcode:
-        error = SSLError(errno, msg)
-    else:
-        error = SSLError(msg)
-    error.reason = reason_str if reason_str else None
-    error.library = lib_str if lib_str else None
-    return error
+def ssl_error(errstr, errcode=0):
+    if errstr is None:
+        errcode = lib.ERR_peek_last_error()
+    try:
+        return fill_sslerror(SSLError, errcode, errstr)
+    finally:
+        lib.ERR_clear_error()
 
 ERR_CODES_TO_NAMES = {}
 ERR_NAMES_TO_CODES = {}
@@ -87,84 +66,85 @@
 for mnemo, number in _lib_codes:
     LIB_CODES_TO_NAMES[number] = mnemo
 
-def _fill_and_raise_ssl_error(error, errcode):
-    pass
-    if errcode != 0:
-        library = lib.ERR_GET_LIB(errcode);
-        reason = lib.ERR_GET_REASON(errcode);
-        key = (library, reason)
-        reason_obj = ERR_CODES_TO_NAMES[key]
-        lib_obj = LIB_CODES_TO_NAMES[library]
-        raise error("[%S: %S]" % (lib_obj, reason_obj))
-
-def _last_error():
-    errcode = lib.ERR_peek_last_error()
-    _fill_and_raise_ssl_error(SSLError, errcode)
-    #buf = ffi.new("char[4096]")
-    #length = lib.ERR_error_string(errcode, buf)
-    #return ffi.string(buf).decode()
-
 
 # the PySSL_SetError equivalent
-def ssl_socket_error(ss, ret):
+def pyssl_error(obj, ret):
     errcode = lib.ERR_peek_last_error()
 
-    if ss is None:
-        return ssl_error(None, errcode=errcode)
-    elif ss.ssl:
-        err = lib.SSL_get_error(ss.ssl, ret)
-    else:
-        err = SSL_ERROR_SSL
     errstr = ""
     errval = 0
     errtype = SSLError
+    e = lib.ERR_peek_last_error()
 
-    if err == SSL_ERROR_ZERO_RETURN:
-        errtype = ZeroReturnError
-        errstr = "TLS/SSL connection has been closed"
-        errval = SSL_ERROR_ZERO_RETURN
-    elif err == SSL_ERROR_WANT_READ:
-        errtype = WantReadError
-        errstr = "The operation did not complete (read)"
-        errval = SSL_ERROR_WANT_READ
-    elif err == SSL_ERROR_WANT_WRITE:
-        errtype = WantWriteError
-        errstr = "The operation did not complete (write)"
-        errval = SSL_ERROR_WANT_WRITE
-    elif err == SSL_ERROR_WANT_X509_LOOKUP:
-        errstr = "The operation did not complete (X509 lookup)"
-        errval = SSL_ERROR_WANT_X509_LOOKUP
-    elif err == SSL_ERROR_WANT_CONNECT:
-        errstr = "The operation did not complete (connect)"
-        errval = SSL_ERROR_WANT_CONNECT
-    elif err == SSL_ERROR_SYSCALL:
-        xxx
-        e = lib.ERR_get_error()
-        if e == 0:
-            if ret == 0 or ss.w_socket() is None:
-                w_errtype = get_error(space).w_EOFError
-                errstr = "EOF occurred in violation of protocol"
-                errval = PY_SSL_ERROR_EOF
-            elif ret == -1:
-                # the underlying BIO reported an I/0 error
-                error = rsocket.last_error()
-                return interp_socket.converted_error(space, error)
+    if obj.ssl != ffi.NULL:
+        err = lib.SSL_get_error(obj.ssl, ret)
+
+        if err == SSL_ERROR_ZERO_RETURN:
+            errtype = SSLZeroReturnError
+            errstr = "TLS/SSL connection has been closed"
+            errval = SSL_ERROR_ZERO_RETURN
+        elif err == SSL_ERROR_WANT_READ:
+            errtype = SSLWantReadError
+            errstr = "The operation did not complete (read)"
+            errval = SSL_ERROR_WANT_READ
+        elif err == SSL_ERROR_WANT_WRITE:
+            errtype = SSLWantWriteError
+            errstr = "The operation did not complete (write)"
+            errval = SSL_ERROR_WANT_WRITE
+        elif err == SSL_ERROR_WANT_X509_LOOKUP:
+            errstr = "The operation did not complete (X509 lookup)"
+            errval = SSL_ERROR_WANT_X509_LOOKUP
+        elif err == SSL_ERROR_WANT_CONNECT:
+            errstr = "The operation did not complete (connect)"
+            errval = SSL_ERROR_WANT_CONNECT
+        elif err == SSL_ERROR_SYSCALL:
+            if e == 0:
+                if ret == 0 or obj.get_socket_or_None() is None:
+                    errtype = EOFError
+                    errstr = "EOF occurred in violation of protocol"
+                    errval = SSL_ERROR_EOF
+                elif ret == -1:
+                    # the underlying BIO reported an I/0 error
+                    errno = ffi.errno
+                    return IOError(errno)
+                else:
+                    errtype = SSLSyscallError
+                    errstr = "Some I/O error occurred"
+                    errval = SSL_ERROR_SYSCALL
             else:
-                w_errtype = get_error(space).w_SyscallError
-                errstr = "Some I/O error occurred"
-                errval = PY_SSL_ERROR_SYSCALL
+                errstr = _str_from_buf(lib.ERR_error_string(e, ffi.NULL))
+                errval = SSL_ERROR_SYSCALL
+        elif err == SSL_ERROR_SSL:
+            errval = SSL_ERROR_SSL
+            if errcode != 0:
+                errstr = _str_from_buf(lib.ERR_error_string(errcode, ffi.NULL))
+            else:
+                errstr = "A failure in the SSL library occurred"
         else:
-            errstr = rffi.charp2str(libssl_ERR_error_string(e, None))
-            errval = PY_SSL_ERROR_SYSCALL
-    elif err == SSL_ERROR_SSL:
-        errval = SSL_ERROR_SSL
-        if errcode != 0:
-            errstr = _str_to_ffi_buffer(lib.ERR_error_string(errcode, ffi.NULL))
-        else:
-            errstr = "A failure in the SSL library occurred"
-    else:
-        errstr = "Invalid error code"
-        errval = SSL_ERROR_INVALID_ERROR_CODE
+            errstr = "Invalid error code"
+            errval = SSL_ERROR_INVALID_ERROR_CODE
+    return fill_sslerror(errtype, errval, errstr, e)
 
-    return errtype(errstr, errval)
 
+def fill_sslerror(errtype, ssl_errno, errstr, errcode):
+    reason_str = None
+    lib_str = None
+    if errcode != 0:
+        err_lib = lib.ERR_GET_LIB(errcode)
+        err_reason = lib.ERR_GET_REASON(errcode)
+        reason_str = ERR_CODES_TO_NAMES.get((err_lib, err_reason), None)
+        lib_str = LIB_CODES_TO_NAMES.get(err_lib, None)
+        if errstr is None:
+            errstr = _str_from_buf(lib.ERR_reason_error_string(errcode))
+    if not errstr:
+        msg = "unknown error"
+    if reason_str and lib_str:
+        msg = "[%s: %s] %s" % (lib_str, reason_str, errstr)
+    elif lib_str:
+        msg = "[%s] %s" % (lib_str, errstr)
+
+    err_value = errtype(ssl_errno, msg)
+    err_value.reason = reason_str if reason_str else None
+    err_value.library = lib_str if lib_str else None
+    return err_value
+
diff --git a/lib_pypy/openssl/_stdssl/utility.py b/lib_pypy/openssl/_stdssl/utility.py
--- a/lib_pypy/openssl/_stdssl/utility.py
+++ b/lib_pypy/openssl/_stdssl/utility.py
@@ -33,3 +33,6 @@
         else:
             return ffi.from_buffer(view)
 
+def _str_from_buf(buf):
+    return ffi.string(buf).decode('utf-8')
+
diff --git a/lib_pypy/ssl.py b/lib_pypy/ssl.py
--- a/lib_pypy/ssl.py
+++ b/lib_pypy/ssl.py
@@ -1,9 +1,3 @@
-# This file exposes the Standard Library API for the ssl module
-
-#from openssl._stdssl import (_PROTOCOL_NAMES, _OPENSSL_API_VERSION)
-#from openssl._stdssl import _ssl
-#from openssl._stdssl import *
-
 # Wrapper module for _ssl, providing some additional facilities
 # implemented in Python.  Written by Bill Janssen.
 
@@ -151,6 +145,7 @@
 from socket import SOL_SOCKET, SO_TYPE
 import base64        # for DER-to-PEM translation
 import errno
+import warnings
 
 
 socket_error = OSError  # keep that public name in module namespace
@@ -411,12 +406,16 @@
 
     def _load_windows_store_certs(self, storename, purpose):
         certs = bytearray()
-        for cert, encoding, trust in enum_certificates(storename):
-            # CA certs are never PKCS#7 encoded
-            if encoding == "x509_asn":
-                if trust is True or purpose.oid in trust:
-                    certs.extend(cert)
-        self.load_verify_locations(cadata=certs)
+        try:
+            for cert, encoding, trust in enum_certificates(storename):
+                # CA certs are never PKCS#7 encoded
+                if encoding == "x509_asn":
+                    if trust is True or purpose.oid in trust:
+                        certs.extend(cert)
+        except PermissionError:
+            warnings.warn("unable to enumerate Windows certificate store")
+        if certs:
+            self.load_verify_locations(cadata=certs)
         return certs
 
     def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
@@ -566,7 +565,7 @@
         server hostame is set."""
         return self._sslobj.server_hostname
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to 'len' bytes from the SSL object and return them.
 
         If 'buffer' is provided, read into this buffer and return the number of
@@ -575,7 +574,7 @@
         if buffer is not None:
             v = self._sslobj.read(len, buffer)
         else:
-            v = self._sslobj.read(len or 1024)
+            v = self._sslobj.read(len)
         return v
 
     def write(self, data):
@@ -781,7 +780,7 @@
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
 

From pypy.commits at gmail.com  Tue Nov 15 11:56:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 08:56:18 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: to-do
Message-ID: <582b3e32.c64bc20a.f290.bdb4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5747:5ecd9f55400b
Date: 2016-11-15 17:56 +0100
http://bitbucket.org/pypy/extradoc/changeset/5ecd9f55400b/

Log:	to-do

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -41,6 +41,9 @@
   
 * PEP 475: Retry system calls failing with EINTR
 
+* ast compiler: clean up POP_EXCEPT: either remove them, or use it to clean up
+  the "finally: name = None; del name" nonsense at the end of any except block
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Tue Nov 15 12:32:51 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Nov 2016 09:32:51 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-injection: fix tests for -A
Message-ID: <582b46c3.8c1f1c0a.21079.bacd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-injection
Changeset: r88389:c5f1e4703974
Date: 2016-11-15 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/c5f1e4703974/

Log:	fix tests for -A

diff --git a/pypy/module/cpyext/injection/test/test_injection.py b/pypy/module/cpyext/injection/test/test_injection.py
--- a/pypy/module/cpyext/injection/test/test_injection.py
+++ b/pypy/module/cpyext/injection/test/test_injection.py
@@ -1,10 +1,13 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.pyobject import _has_a_pyobj
 from pypy.interpreter.gateway import interp2app
+from pypy.conftest import option
 
 class AppTestTypeObject(AppTestCpythonExtensionBase):
 
     def setup_class(cls):
+        if option.runappdirect:
+            skip('untranslated only')
         def is_there_an_pyobj_version(space, w_obj):
             if _has_a_pyobj(space, w_obj):
                 return space.w_True
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -233,7 +233,7 @@
         else:
             def w_import_module(self, name, init=None, body='', filename=None,
                     include_dirs=None, PY_SSIZE_T_CLEAN=False):
-                from extbuild import get_sys_info_app
+                from pypy.tool.cpyext.extbuild import get_sys_info_app
                 sys_info = get_sys_info_app(self.udir)
                 return sys_info.import_module(
                     name, init=init, body=body, filename=filename,
@@ -243,7 +243,7 @@
 
             def w_import_extension(self, modname, functions, prologue="",
                 include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
-                from extbuild import get_sys_info_app
+                from pypy.tool.cpyext.extbuild import get_sys_info_app
                 sys_info = get_sys_info_app(self.udir)
                 return sys_info.import_extension(
                     modname, functions, prologue=prologue,
@@ -253,14 +253,14 @@
 
             def w_compile_module(self, name,
                     source_files=None, source_strings=None):
-                from extbuild import get_sys_info_app
+                from pypy.tool.cpyext.extbuild import get_sys_info_app
                 sys_info = get_sys_info_app(self.udir)
                 return sys_info.compile_extension_module(name,
                     source_files=source_files, source_strings=source_strings)
             cls.w_compile_module = w_compile_module
 
             def w_load_module(self, mod, name):
-                from extbuild import get_sys_info_app
+                from pypy.tool.cpyext.extbuild import get_sys_info_app
                 sys_info = get_sys_info_app(self.udir)
                 return sys_info.load_module(mod, name)
             cls.w_load_module = w_load_module

From pypy.commits at gmail.com  Tue Nov 15 12:32:53 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Nov 2016 09:32:53 -0800 (PST)
Subject: [pypy-commit] pypy cpyext-injection: tweaks for downstream packages
Message-ID: <582b46c5.28c9c20a.cf743.d550@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-injection
Changeset: r88390:7da08ee4b6be
Date: 2016-11-15 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/7da08ee4b6be/

Log:	tweaks for downstream packages

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -66,6 +66,7 @@
     g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['OPT'] = "" 
 
     global _config_vars
     _config_vars = g
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -39,6 +39,10 @@
 #define PY_SIZE_MAX ((size_t)-1)
 #endif
 
+/* CPython needs this for the c-extension datetime, which is pure python on PyPy 
+   downstream packages assume it is here (Pandas for instance) */
+#include <time.h> 
+
 /* uintptr_t is the C9X name for an unsigned integral type such that a
  * legitimate void* can be cast to uintptr_t and then back to void* again
  * without loss of information.  Similarly for intptr_t, wrt a signed

From pypy.commits at gmail.com  Tue Nov 15 12:42:18 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Nov 2016 09:42:18 -0800 (PST)
Subject: [pypy-commit] pypy default: tweaks for downstream packages
Message-ID: <582b48fa.0d1a1c0a.d13e0.baf2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88391:ae0c7c00553c
Date: 2016-11-15 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ae0c7c00553c/

Log:	tweaks for downstream packages

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -66,6 +66,7 @@
     g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['OPT'] = "" 
 
     global _config_vars
     _config_vars = g
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -39,6 +39,10 @@
 #define PY_SIZE_MAX ((size_t)-1)
 #endif
 
+/* CPython needs this for the c-extension datetime, which is pure python on PyPy 
+   downstream packages assume it is here (Pandas for instance) */
+#include <time.h> 
+
 /* uintptr_t is the C9X name for an unsigned integral type such that a
  * legitimate void* can be cast to uintptr_t and then back to void* again
  * without loss of information.  Similarly for intptr_t, wrt a signed

From pypy.commits at gmail.com  Tue Nov 15 12:42:20 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 15 Nov 2016 09:42:20 -0800 (PST)
Subject: [pypy-commit] pypy default: document the get_config_vars differences
Message-ID: <582b48fc.c4251c0a.ded54.bf93@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88392:f3538b9e04af
Date: 2016-11-15 19:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f3538b9e04af/

Log:	document the get_config_vars differences

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -473,6 +473,10 @@
   time and the standard deviation, instead of the minimum, since the minimum is
   often misleading.
 
+* The ``get_config_vars`` method of ``sysconfig`` and ``distutils.sysconfig``
+  are not complete. On POSIX platforms, CPython fishes configuration variables
+  from the Makefile used to build the interpreter. PyPy should bake the values
+  in during compilation, but does not do that yet.
 
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html

From pypy.commits at gmail.com  Tue Nov 15 13:13:47 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 15 Nov 2016 10:13:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: (arigo, cfbolz,
 richard around):
Message-ID: <582b505b.06891c0a.12730.d6f5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88393:195ff4d7c074
Date: 2016-11-15 18:23 +0000
http://bitbucket.org/pypy/pypy/changeset/195ff4d7c074/

Log:	(arigo, cfbolz, richard around):

	in progress: start a refactoring to store sys_exc_info on the ec,
	instead of having last_exception on each frame. this makes it more
	like CPython and should make it possible to not force every frame
	when an exception is raised.

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -333,13 +333,12 @@
             tb = tb.next
         self._application_traceback = tb
 
-    def record_context(self, space, frame):
-        """Record a __context__ for this exception if one exists,
-        searching from the current frame.
+    def record_context(self, space, ec):
+        """Record a __context__ for this exception if one exists.
         """
         if self._context_recorded:
             return
-        last = frame._exc_info_unroll(space)
+        last = ec.sys_exc_info()
         try:
             if last is not None:
                 self.chain_exceptions(space, last)
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -28,6 +28,10 @@
     def __init__(self, space):
         self.space = space
         self.topframeref = jit.vref_None
+        # this is exposed to app-level as 'sys.exc_info()'.  At any point in
+        # time it is the exception caught by the topmost 'except ... as e:'
+        # app-level block.
+        self.sys_exc_operror = None
         self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
@@ -230,23 +234,10 @@
         # NOTE: the result is not the wrapped sys.exc_info() !!!
 
         """
-        return self.gettopframe()._exc_info_unroll(self.space)
+        return self.sys_exc_operror
 
     def set_sys_exc_info(self, operror):
-        frame = self.gettopframe_nohidden()
-        if frame:     # else, the exception goes nowhere and is lost
-            frame.last_exception = operror
-
-    def clear_sys_exc_info(self):
-        # Find the frame out of which sys_exc_info() would return its result,
-        # and hack this frame's last_exception to become the cleared
-        # OperationError (which is different from None!).
-        frame = self.gettopframe_nohidden()
-        while frame:
-            if frame.last_exception is not None:
-                frame.last_exception = get_cleared_operation_error(self.space)
-                break
-            frame = self.getnextframe_nohidden(frame)
+        self.sys_exc_operror = operror
 
     @jit.dont_look_inside
     def settrace(self, w_func):
@@ -356,7 +347,6 @@
                     event == 'c_exception'):
                 return
 
-            last_exception = frame.last_exception
             if event == 'leaveframe':
                 event = 'return'
 
@@ -372,7 +362,6 @@
                     raise
 
             finally:
-                frame.last_exception = last_exception
                 self.is_tracing -= 1
 
     def checksignals(self):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,6 @@
     f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
     f_generator_nowref       = None               # (only one of the two attrs)
     last_instr               = -1
-    last_exception           = None
     f_backref                = jit.vref_None
     
     escaped                  = False  # see mark_as_escaped()
@@ -328,10 +327,6 @@
                                             executioncontext)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
-            # it used to say self.last_exception = None
-            # this is now done by the code in pypyjit module
-            # since we don't want to invalidate the virtualizable
-            # for no good reason
             got_exception = False
         finally:
             executioncontext.leave(self, w_exitvalue, got_exception)
@@ -882,33 +877,6 @@
     def fdel_f_trace(self, space):
         self.getorcreatedebug().w_f_trace = None
 
-    def fget_f_exc_type(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return f.last_exception.w_type
-        return space.w_None
-
-    def fget_f_exc_value(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return f.last_exception.get_w_value(space)
-        return space.w_None
-
-    def fget_f_exc_traceback(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return space.wrap(f.last_exception.get_traceback())
-        return space.w_None
-
     def fget_f_restricted(self, space):
         if space.config.objspace.honor__builtins__:
             return space.wrap(self.builtin is not space.builtin)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -73,7 +73,7 @@
         try:
             next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
         except OperationError as operr:
-            operr.record_context(self.space, self)
+            operr.record_context(self.space, ec)
             next_instr = self.handle_operation_error(ec, operr)
         except RaiseWithExplicitTraceback as e:
             next_instr = self.handle_operation_error(ec, e.operr,
@@ -688,12 +688,11 @@
         if nbargs > 2:
             raise BytecodeCorruption("bad RAISE_VARARGS oparg")
         if nbargs == 0:
-            last_operr = self._exc_info_unroll(space, for_hidden=True)
+            last_operr = self.space.getexecutioncontext().sys_exc_info()
             if last_operr is None:
                 raise oefmt(space.w_RuntimeError,
                             "No active exception to reraise")
             # re-raise, no new traceback obj will be attached
-            self.last_exception = last_operr
             raise RaiseWithExplicitTraceback(last_operr)
         if nbargs == 2:
             w_cause = self.popvalue()
@@ -747,14 +746,20 @@
             self.setdictscope(w_locals)
 
     def POP_EXCEPT(self, oparg, next_instr):
-        block = self.pop_block()
-        block.cleanup(self)
-        return
+        pass    # no-op for now: done by the END_FINALLY that follows anyway
 
     def POP_BLOCK(self, oparg, next_instr):
         block = self.pop_block()
         block.cleanup(self)  # the block knows how to clean up the value stack
 
+    def save_and_change_sys_exc_info(self, operationerr):
+        ec = self.space.getexecutioncontext()
+        last_exception = ec.sys_exc_info()
+        block = SysExcInfoRestorer(last_exception, self.lastblock)
+        self.lastblock = block
+        if operationerr is not None:   # otherwise, don't change sys_exc_info
+            ec.set_sys_exc_info(operationerr)
+
     def end_finally(self):
         # unlike CPython, there are two statically distinct cases: the
         # END_FINALLY might be closing an 'except' block or a 'finally'
@@ -763,8 +768,14 @@
         #   [exception value we are now handling]
         #   [wrapped SApplicationException]
         # In the case of a finally: block, the stack contains only one
-        # item (unlike CPython which can have 1, 2 or 3 items):
+        # item (unlike CPython which can have 1, 2, 3 or 5 items, and
+        # even in one case a non-fixed number of items):
         #   [wrapped subclass of SuspendedUnroller]
+
+        block = self.pop_block()
+        assert isinstance(block, SysExcInfoRestorer)
+        block.cleanupstack()   # restores ec.sys_exc_operror
+
         w_top = self.popvalue()
         if self.space.is_w(w_top, self.space.w_None):
             # case of a finally: block with no exception
@@ -1137,8 +1148,8 @@
         w_exit = self.space.get(w_descr, w_manager)
         self.settopvalue(w_exit)
         w_result = self.space.get_and_call_function(w_enter, w_manager)
-        block = WithBlock(self.valuestackdepth,
-                          next_instr + offsettoend, self.lastblock)
+        block = FinallyBlock(self.valuestackdepth,
+                             next_instr + offsettoend, self.lastblock)
         self.lastblock = block
         self.pushvalue(w_result)
 
@@ -1150,6 +1161,7 @@
         if isinstance(w_unroller, SApplicationException):
             # app-level exception
             operr = w_unroller.operr
+            # this looks again like the kind of code we have. except that it's a call, not a block
             old_last_exception = self.last_exception
             self.last_exception = operr
             w_traceback = self.space.wrap(operr.get_traceback())
@@ -1183,6 +1195,7 @@
             if self.space.is_true(w_suppress):
                 # __exit__() returned True -> Swallow the exception.
                 self.settopvalue(self.space.w_None)
+        # this is always followed by END_FINALLY
 
     @jit.unroll_safe
     def call_function(self, oparg, w_starstar=None, has_vararg=False):
@@ -1680,28 +1693,26 @@
             return r_uint(self.handlerposition)
 
 
-class ExceptHandlerBlock(FrameBlock):
+class SysExcInfoRestorer(FrameBlock):
     """
-    This is a special, implicit block type which is created when entering an
-    except handler. It does not belong to any opcode
+    This is a special, implicit block type which is created when entering a
+    finally or except handler. It does not belong to any opcode
     """
 
     _immutable_ = True
-    _opname = 'EXCEPT_HANDLER_BLOCK' # it's not associated to any opcode
+    _opname = 'SYS_EXC_INFO_RESTORER' # it's not associated to any opcode
     handling_mask = 0 # this block is never handled, only popped by POP_EXCEPT
 
+    def __init__(self, operr, previous):
+        self.operr = operr
+        self.previous = previous
+
     def handle(self, frame, unroller):
         assert False # never called
 
     def cleanupstack(self, frame):
-        frame.dropvaluesuntil(self.valuestackdepth+1)
-        w_last_exception = frame.popvalue()
-        if not isinstance(w_last_exception, W_OperationError):
-            msg = "expected an OperationError, got %s" % (
-                frame.space.str_w(w_last_exception))
-            raise BytecodeCorruption(msg)
-        frame.last_exception = w_last_exception.operr
-        FrameBlock.cleanupstack(self, frame)
+        ec = frame.space.getexecutioncontext()
+        ec.set_sys_exc_info(self.operr)
 
 
 class ExceptBlock(FrameBlock):
@@ -1715,22 +1726,18 @@
         # push the exception to the value stack for inspection by the
         # exception handler (the code after the except:)
         self.cleanupstack(frame)
+        # the stack setup is slightly different than in CPython:
+        # instead of the traceback, we store the unroller object,
+        # wrapped.
         assert isinstance(unroller, SApplicationException)
         operationerr = unroller.operr
         operationerr.normalize_exception(frame.space)
-        # the stack setup is slightly different than in CPython:
-        # instead of the traceback, we store the unroller object,
-        # wrapped.
-        # this is popped by POP_EXCEPT, which is present only in py3k
-        w_last_exception = W_OperationError(frame.last_exception)
-        w_last_exception = frame.space.wrap(w_last_exception)
-        frame.pushvalue(w_last_exception)
-        block = ExceptHandlerBlock(self.valuestackdepth, 0, frame.lastblock)
-        frame.lastblock = block
         frame.pushvalue(frame.space.wrap(unroller))
         frame.pushvalue(operationerr.get_w_value(frame.space))
         frame.pushvalue(operationerr.w_type)
-        frame.last_exception = operationerr
+        # set the current value of sys_exc_info to operationerr,
+        # saving the old value in a custom type of FrameBlock
+        frame.save_and_change_sys_exc_info(operationerr)
         return r_uint(self.handlerposition)   # jump to the handler
 
 
@@ -1740,7 +1747,6 @@
     _immutable_ = True
     _opname = 'SETUP_FINALLY'
     handling_mask = -1     # handles every kind of SuspendedUnroller
-    restore_last_exception = True # set to False by WithBlock
 
     def handle(self, frame, unroller):
         # any abnormal reason for unrolling a finally: triggers the end of
@@ -1752,53 +1758,44 @@
             operationerr = unroller.operr
             operationerr.normalize_exception(frame.space)
         frame.pushvalue(frame.space.wrap(unroller))
-        if operationerr and self.restore_last_exception:
-            frame.last_exception = operationerr
+        # set the current value of sys_exc_info to operationerr,
+        # saving the old value in a custom type of FrameBlock
+        frame.save_and_change_sys_exc_info(operationerr)
         return r_uint(self.handlerposition)   # jump to the handler
 
 
-class WithBlock(FinallyBlock):
-
-    _immutable_ = True
-    restore_last_exception = False
-
-    def handle(self, frame, unroller):
-        if isinstance(unroller, SApplicationException):
-            unroller.operr.normalize_exception(frame.space)
-        return FinallyBlock.handle(self, frame, unroller)
-
-block_classes = {'EXCEPT_HANDLER_BLOCK': ExceptHandlerBlock,
+block_classes = {'SYS_EXC_INFO_RESTORER': SysExcInfoRestorer,
                  'SETUP_LOOP': LoopBlock,
                  'SETUP_EXCEPT': ExceptBlock,
                  'SETUP_FINALLY': FinallyBlock,
-                 'SETUP_WITH': WithBlock,
+                 'SETUP_WITH': FinallyBlock,
                  }
 
 
-class W_OperationError(W_Root):
-    """
-    Tiny applevel wrapper around an OperationError.
-    """
-
-    def __init__(self, operr):
-        self.operr = operr
-
-    def descr_reduce(self, space):
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod = space.getbuiltinmodule('_pickle_support')
-        mod = space.interp_w(MixedModule, w_mod)
-        w_new_inst = mod.get('operationerror_new')
-        w_args = space.newtuple([])
-        operr = self.operr
-        if operr is None:
-            return space.newtuple([w_new_inst, w_args])
-        w_state = space.newtuple([operr.w_type, operr.get_w_value(space),
-                                  operr.get_traceback()])
-        return space.newtuple([w_new_inst, w_args, w_state])
-
-    def descr_setstate(self, space, w_state):
-        w_type, w_value, w_tb = space.fixedview(w_state, 3)
-        self.operr = OperationError(w_type, w_value, w_tb)
+##class W_OperationError(W_Root):
+##    """
+##    Tiny applevel wrapper around an OperationError.
+##    """
+##
+##    def __init__(self, operr):
+##        self.operr = operr
+##
+##    def descr_reduce(self, space):
+##        from pypy.interpreter.mixedmodule import MixedModule
+##        w_mod = space.getbuiltinmodule('_pickle_support')
+##        mod = space.interp_w(MixedModule, w_mod)
+##        w_new_inst = mod.get('operationerror_new')
+##        w_args = space.newtuple([])
+##        operr = self.operr
+##        if operr is None:
+##            return space.newtuple([w_new_inst, w_args])
+##        w_state = space.newtuple([operr.w_type, operr.get_w_value(space),
+##                                  operr.get_traceback()])
+##        return space.newtuple([w_new_inst, w_args, w_state])
+##
+##    def descr_setstate(self, space, w_state):
+##        w_type, w_value, w_tb = space.fixedview(w_state, 3)
+##        self.operr = OperationError(w_type, w_value, w_tb)
 
 def source_as_str(space, w_source, funcname, what, flags):
     """Return source code as str0 with adjusted compiler flags
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -464,7 +464,7 @@
 from pypy.interpreter.eval import Code
 from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS
 from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pyopcode import SuspendedUnroller, W_OperationError
+from pypy.interpreter.pyopcode import SuspendedUnroller
 from pypy.interpreter.module import Module
 from pypy.interpreter.function import (Function, Method, StaticMethod,
     ClassMethod, BuiltinFunction, descr_function_get)
@@ -616,9 +616,6 @@
     f_lasti = GetSetProperty(PyFrame.fget_f_lasti),
     f_trace = GetSetProperty(PyFrame.fget_f_trace, PyFrame.fset_f_trace,
                              PyFrame.fdel_f_trace),
-    f_exc_type = GetSetProperty(PyFrame.fget_f_exc_type),
-    f_exc_value = GetSetProperty(PyFrame.fget_f_exc_value),
-    f_exc_traceback = GetSetProperty(PyFrame.fget_f_exc_traceback),
     f_restricted = GetSetProperty(PyFrame.fget_f_restricted),
     f_code = GetSetProperty(PyFrame.fget_code),
     f_locals = GetSetProperty(PyFrame.fget_getdictscope),
@@ -878,8 +875,8 @@
 SuspendedUnroller.typedef = TypeDef("SuspendedUnroller")
 SuspendedUnroller.typedef.acceptable_as_base_class = False
 
-W_OperationError.typedef = TypeDef("OperationError",
-    __reduce__ = interp2app(W_OperationError.descr_reduce),
-    __setstate__ = interp2app(W_OperationError.descr_setstate),
-)
-W_OperationError.typedef.acceptable_as_base_class = False
+## W_OperationError.typedef = TypeDef("OperationError",
+##     __reduce__ = interp2app(W_OperationError.descr_reduce),
+##     __setstate__ = interp2app(W_OperationError.descr_setstate),
+## )
+## W_OperationError.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Tue Nov 15 14:31:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 11:31:26 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: copy upstream pytest-2.9.2 and
 py-1.4.29
Message-ID: <582b628e.54b31c0a.68a30.e69b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88394:f054ccfda623
Date: 2016-11-15 01:46 +0000
http://bitbucket.org/pypy/pypy/changeset/f054ccfda623/

Log:	copy upstream pytest-2.9.2 and py-1.4.29

diff too long, truncating to 2000 out of 12680 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls
-
-
-class Interpretable(View):
-    """A parse tree node with a few extra methods."""
-    explanation = None
-
-    def is_builtin(self, frame):
-        return False
-
-    def eval(self, frame):
-        # fall-back for unknown expression nodes
-        try:
-            expr = ast.Expression(self.__obj__)
-            expr.filename = '<eval>'
-            self.__obj__.filename = '<eval>'
-            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
-            result = frame.eval(co)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        self.result = result
-        self.explanation = self.explanation or frame.repr(self.result)
-
-    def run(self, frame):
-        # fall-back for unknown statement nodes
-        try:
-            expr = ast.Module(None, ast.Stmt([self.__obj__]))
-            expr.filename = '<run>'
-            co = pycodegen.ModuleCodeGenerator(expr).getCode()
-            frame.exec_(co)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-
-    def nice_explanation(self):
-        return format_explanation(self.explanation)
-
-
-class Name(Interpretable):
-    __view__ = ast.Name
-
-    def is_local(self, frame):
-        source = '%r in locals() is not globals()' % self.name
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def is_global(self, frame):
-        source = '%r in globals()' % self.name
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def is_builtin(self, frame):
-        source = '%r not in locals() and %r not in globals()' % (
-            self.name, self.name)
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def eval(self, frame):
-        super(Name, self).eval(frame)
-        if not self.is_local(frame):
-            self.explanation = self.name
-
-class Compare(Interpretable):
-    __view__ = ast.Compare
-
-    def eval(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        for operation, expr2 in self.ops:
-            if hasattr(self, 'result'):
-                # shortcutting in chained expressions
-                if not frame.is_true(self.result):
-                    break
-            expr2 = Interpretable(expr2)
-            expr2.eval(frame)
-            self.explanation = "%s %s %s" % (
-                expr.explanation, operation, expr2.explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % operation
-            try:
-                self.result = frame.eval(source,
-                                         __exprinfo_left=expr.result,
-                                         __exprinfo_right=expr2.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-            expr = expr2
-
-class And(Interpretable):
-    __view__ = ast.And
-
-    def eval(self, frame):
-        explanations = []
-        for expr in self.nodes:
-            expr = Interpretable(expr)
-            expr.eval(frame)
-            explanations.append(expr.explanation)
-            self.result = expr.result
-            if not frame.is_true(expr.result):
-                break
-        self.explanation = '(' + ' and '.join(explanations) + ')'
-
-class Or(Interpretable):
-    __view__ = ast.Or
-
-    def eval(self, frame):
-        explanations = []
-        for expr in self.nodes:
-            expr = Interpretable(expr)
-            expr.eval(frame)
-            explanations.append(expr.explanation)
-            self.result = expr.result
-            if frame.is_true(expr.result):
-                break
-        self.explanation = '(' + ' or '.join(explanations) + ')'
-
-
-# == Unary operations ==
-keepalive = []
-for astclass, astpattern in {
-    ast.Not    : 'not __exprinfo_expr',
-    ast.Invert : '(~__exprinfo_expr)',
-    }.items():
-
-    class UnaryArith(Interpretable):
-        __view__ = astclass
-
-        def eval(self, frame, astpattern=astpattern):
-            expr = Interpretable(self.expr)
-            expr.eval(frame)
-            self.explanation = astpattern.replace('__exprinfo_expr',
-                                                  expr.explanation)
-            try:
-                self.result = frame.eval(astpattern,
-                                         __exprinfo_expr=expr.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-    keepalive.append(UnaryArith)
-
-# == Binary operations ==
-for astclass, astpattern in {
-    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
-    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
-    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
-    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
-    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
-    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
-    }.items():
-
-    class BinaryArith(Interpretable):
-        __view__ = astclass
-
-        def eval(self, frame, astpattern=astpattern):
-            left = Interpretable(self.left)
-            left.eval(frame)
-            right = Interpretable(self.right)
-            right.eval(frame)
-            self.explanation = (astpattern
-                                .replace('__exprinfo_left',  left .explanation)
-                                .replace('__exprinfo_right', right.explanation))
-            try:
-                self.result = frame.eval(astpattern,
-                                         __exprinfo_left=left.result,
-                                         __exprinfo_right=right.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-    keepalive.append(BinaryArith)
-
-
-class CallFunc(Interpretable):
-    __view__ = ast.CallFunc
-
-    def is_bool(self, frame):
-        source = 'isinstance(__exprinfo_value, bool)'
-        try:
-            return frame.is_true(frame.eval(source,
-                                            __exprinfo_value=self.result))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def eval(self, frame):
-        node = Interpretable(self.node)
-        node.eval(frame)
-        explanations = []
-        vars = {'__exprinfo_fn': node.result}
-        source = '__exprinfo_fn('
-        for a in self.args:
-            if isinstance(a, ast.Keyword):
-                keyword = a.name
-                a = a.expr
-            else:
-                keyword = None
-            a = Interpretable(a)
-            a.eval(frame)
-            argname = '__exprinfo_%d' % len(vars)
-            vars[argname] = a.result
-            if keyword is None:
-                source += argname + ','
-                explanations.append(a.explanation)
-            else:
-                source += '%s=%s,' % (keyword, argname)
-                explanations.append('%s=%s' % (keyword, a.explanation))
-        if self.star_args:
-            star_args = Interpretable(self.star_args)
-            star_args.eval(frame)
-            argname = '__exprinfo_star'
-            vars[argname] = star_args.result
-            source += '*' + argname + ','
-            explanations.append('*' + star_args.explanation)
-        if self.dstar_args:
-            dstar_args = Interpretable(self.dstar_args)
-            dstar_args.eval(frame)
-            argname = '__exprinfo_kwds'
-            vars[argname] = dstar_args.result
-            source += '**' + argname + ','
-            explanations.append('**' + dstar_args.explanation)
-        self.explanation = "%s(%s)" % (
-            node.explanation, ', '.join(explanations))
-        if source.endswith(','):
-            source = source[:-1]
-        source += ')'
-        try:
-            self.result = frame.eval(source, **vars)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        if not node.is_builtin(frame) or not self.is_bool(frame):
-            r = frame.repr(self.result)
-            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-class Getattr(Interpretable):
-    __view__ = ast.Getattr
-
-    def eval(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        source = '__exprinfo_expr.%s' % self.attrname
-        try:
-            self.result = frame.eval(source, __exprinfo_expr=expr.result)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
-        # if the attribute comes from the instance, its value is interesting
-        source = ('hasattr(__exprinfo_expr, "__dict__") and '
-                  '%r in __exprinfo_expr.__dict__' % self.attrname)
-        try:
-            from_instance = frame.is_true(
-                frame.eval(source, __exprinfo_expr=expr.result))
-        except passthroughex:
-            raise
-        except:
-            from_instance = True
-        if from_instance:
-            r = frame.repr(self.result)
-            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-# == Re-interpretation of full statements ==
-
-class Assert(Interpretable):
-    __view__ = ast.Assert
-
-    def run(self, frame):
-        test = Interpretable(self.test)
-        test.eval(frame)
-        # print the result as  'assert <explanation>'
-        self.result = test.result
-        self.explanation = 'assert ' + test.explanation
-        if not frame.is_true(test.result):
-            try:
-                raise BuiltinAssertionError
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-class Assign(Interpretable):
-    __view__ = ast.Assign
-
-    def run(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        self.result = expr.result
-        self.explanation = '... = ' + expr.explanation
-        # fall-back-run the rest of the assignment
-        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
-        mod = ast.Module(None, ast.Stmt([ass]))
-        mod.filename = '<run>'
-        co = pycodegen.ModuleCodeGenerator(mod).getCode()
-        try:
-            frame.exec_(co, __exprinfo_expr=expr.result)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-
-class Discard(Interpretable):
-    __view__ = ast.Discard
-
-    def run(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        self.result = expr.result
-        self.explanation = expr.explanation
-
-class Stmt(Interpretable):
-    __view__ = ast.Stmt
-
-    def run(self, frame):
-        for stmt in self.nodes:
-            stmt = Interpretable(stmt)
-            stmt.run(frame)
-
-
-def report_failure(e):
-    explanation = e.node.nice_explanation()
-    if explanation:
-        explanation = ", in: " + explanation
-    else:
-        explanation = ""
-    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
-
-def check(s, frame=None):
-    if frame is None:
-        frame = sys._getframe(1)
-        frame = py.code.Frame(frame)
-    expr = parse(s, 'eval')
-    assert isinstance(expr, ast.Expression)
-    node = Interpretable(expr.node)
-    try:
-        node.eval(frame)
-    except passthroughex:
-        raise
-    except Failure:
-        e = sys.exc_info()[1]
-        report_failure(e)
-    else:
-        if not frame.is_true(node.result):
-            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
-
-
-###########################################################
-# API / Entry points
-# #########################################################
-
-def interpret(source, frame, should_fail=False):
-    module = Interpretable(parse(source, 'exec').node)
-    #print "got module", module
-    if isinstance(frame, py.std.types.FrameType):
-        frame = py.code.Frame(frame)
-    try:
-        module.run(frame)
-    except Failure:
-        e = sys.exc_info()[1]
-        return getfailure(e)
-    except passthroughex:
-        raise
-    except:
-        import traceback
-        traceback.print_exc()
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-    else:
-        return None
-
-def getmsg(excinfo):
-    if isinstance(excinfo, tuple):
-        excinfo = py.code.ExceptionInfo(excinfo)
-    #frame, line = gettbline(tb)
-    #frame = py.code.Frame(frame)
-    #return interpret(line, frame)
-
-    tb = excinfo.traceback[-1]
-    source = str(tb.statement).strip()
-    x = interpret(source, tb.frame, should_fail=True)
-    if not isinstance(x, str):
-        raise TypeError("interpret returned non-string %r" % (x,))
-    return x
-
-def getfailure(e):
-    explanation = e.node.nice_explanation()
-    if str(e.value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (e.value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.exc.__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-def run(s, frame=None):
-    if frame is None:
-        frame = sys._getframe(1)
-        frame = py.code.Frame(frame)
-    module = Interpretable(parse(s, 'exec').node)
-    try:
-        module.run(frame)
-    except Failure:
-        e = sys.exc_info()[1]
-        report_failure(e)
-
-
-if __name__ == '__main__':
-    # example:
-    def f():
-        return 5
-
-    def g():
-        return 3
-
-    def h(x):
-        return 'never'
-
-    check("f() * g() == 5")
-    check("not f()")
-    check("not (f() and g() or 0)")
-    check("f() == g()")
-    i = 4
-    check("i == f()")
-    check("len(f()) == 0")
-    check("isinstance(2+3+4, float)")
-
-    run("x = i")
-    check("x == 5")
-
-    run("assert not f(), 'oops'")
-    run("a, b, c = 1, 2")
-    run("a, b, c = f()")
-
-    check("max([f(),g()]) == 4")
-    check("'hello'[g()] == 'h'")
-    run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,12 +1,18 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+"""
+import ast
 import sys
+
+import _pytest._code
 import py
-from _pytest.assertion.util import BuiltinAssertionError
+from _pytest.assertion import util
 u = py.builtin._totext
 
 
-class AssertionError(BuiltinAssertionError):
+class AssertionError(util.BuiltinAssertionError):
     def __init__(self, *args):
-        BuiltinAssertionError.__init__(self, *args)
+        util.BuiltinAssertionError.__init__(self, *args)
         if args:
             # on Python2.6 we get len(args)==2 for: assert 0, (x,y)
             # on Python2.7 and above we always get len(args) == 1
@@ -22,7 +28,7 @@
                     "<[broken __repr__] %s at %0xd>"
                     % (toprint.__class__, id(toprint)))
         else:
-            f = py.code.Frame(sys._getframe(1))
+            f = _pytest._code.Frame(sys._getframe(1))
             try:
                 source = f.code.fullsource
                 if source is not None:
@@ -45,10 +51,357 @@
 
 if sys.version_info > (3, 0):
     AssertionError.__module__ = "builtins"
-    reinterpret_old = "old reinterpretation not available for py3"
+
+if sys.platform.startswith("java"):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
 else:
-    from _pytest.assertion.oldinterpret import interpret as reinterpret_old
-if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
-    from _pytest.assertion.newinterpret import interpret as reinterpret
-else:
-    reinterpret = reinterpret_old
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+try:
+    _Starred = ast.Starred
+except AttributeError:
+    # Python 2. Define a dummy class so isinstance() will always be False.
+    class _Starred(object): pass
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def reinterpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --assert=plain)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = _pytest._code.Frame(sys._getframe(1))
+    return reinterpret(offending_line, frame)
+
+def getfailure(e):
+    explanation = util.format_explanation(e.explanation)
+    value = e.cause[1]
+    if str(value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.cause[0].__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = None
+        if local is None or not self.frame.is_true(local):
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not self.frame.is_true(result):
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        if util._reprcompare is not None:
+            res = util._reprcompare(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            if isinstance(arg, _Starred):
+                arg_name = "__exprinfo_star"
+                ns[arg_name] = arg_result
+                arguments.append("*%s" % (arg_name,))
+                arg_explanations.append("*%s" % (arg_explanation,))
+            else:
+                arg_name = "__exprinfo_%s" % (len(ns),)
+                ns[arg_name] = arg_result
+                arguments.append(arg_name)
+                arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            if keyword.arg:
+                arg_name = "__exprinfo_%s" % (len(ns),)
+                keyword_source = "%s=%%s" % (keyword.arg)
+                arguments.append(keyword_source % (arg_name,))
+                arg_explanations.append(keyword_source % (arg_explanation,))
+            else:
+                arg_name = "__exprinfo_kwds"
+                arguments.append("**%s" % (arg_name,))
+                arg_explanations.append("**%s" % (arg_explanation,))
+
+            ns[arg_name] = arg_result
+
+        if getattr(call, 'starargs', None):
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+
+        if getattr(call, 'kwargs', None):
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            try:
+                result = self.frame.eval(co, __exprinfo_expr=source_result)
+            except AttributeError:
+                # Maybe the attribute name needs to be mangled?
+                if not attr.attr.startswith("__") or attr.attr.endswith("__"):
+                    raise
+                source = "getattr(__exprinfo_expr.__class__, '__name__', '')"
+                co = self._compile(source)
+                class_name = self.frame.eval(co, __exprinfo_expr=source_result)
+                mangled_attr = "_" + class_name +  attr.attr
+                source = "__exprinfo_expr.%s" % (mangled_attr,)
+                co = self._compile(source)
+                result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = None
+        if from_instance is None or self.frame.is_true(from_instance):
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        explanation = "assert %s" % (test_explanation,)
+        if not self.frame.is_true(test_result):
+            try:
+                raise util.BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
+
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -35,6 +35,12 @@
 REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
 ASCII_IS_DEFAULT_ENCODING = sys.version_info[0] < 3
 
+if sys.version_info >= (3,5):
+    ast_Call = ast.Call
+else:
+    ast_Call = lambda a,b,c: ast.Call(a, b, c, None, None)
+
+
 class AssertionRewritingHook(object):
     """PEP302 Import hook which rewrites asserts."""
 
@@ -122,7 +128,7 @@
                     # One of the path components was not a directory, likely
                     # because we're in a zip file.
                     write = False
-                elif e == errno.EACCES:
+                elif e in [errno.EACCES, errno.EROFS, errno.EPERM]:
                     state.trace("read only directory: %r" % fn_pypath.dirname)
                     write = False
                 else:
@@ -131,21 +137,27 @@
         pyc = os.path.join(cache_dir, cache_name)
         # Notice that even if we're in a read-only directory, I'm going
         # to check for a cached pyc. This may not be optimal...
-        co = _read_pyc(fn_pypath, pyc)
+        co = _read_pyc(fn_pypath, pyc, state.trace)
         if co is None:
             state.trace("rewriting %r" % (fn,))
-            co = _rewrite_test(state, fn_pypath)
+            source_stat, co = _rewrite_test(state, fn_pypath)
             if co is None:
                 # Probably a SyntaxError in the test.
                 return None
             if write:
-                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+                _make_rewritten_pyc(state, source_stat, pyc, co)
         else:
             state.trace("found cached rewritten pyc for %r" % (fn,))
         self.modules[name] = co, pyc
         return self
 
     def load_module(self, name):
+        # If there is an existing module object named 'fullname' in
+        # sys.modules, the loader must use that existing module. (Otherwise,
+        # the reload() builtin will not work correctly.)
+        if name in sys.modules:
+            return sys.modules[name]
+
         co, pyc = self.modules.pop(name)
         # I wish I could just call imp.load_compiled here, but __file__ has to
         # be set properly. In Python 3.2+, this all would be handled correctly
@@ -191,14 +203,19 @@
         #  DefaultProvider is appropriate.
         pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider)
 
+    def get_data(self, pathname):
+        """Optional PEP302 get_data API.
+        """
+        with open(pathname, 'rb') as f:
+            return f.read()
 
-def _write_pyc(state, co, source_path, pyc):
+
+def _write_pyc(state, co, source_stat, pyc):
     # Technically, we don't have to have the same pyc format as
     # (C)Python, since these "pycs" should never be seen by builtin
     # import. However, there's little reason deviate, and I hope
     # sometime to be able to use imp.load_compiled to load them. (See
     # the comment in load_module above.)
-    mtime = int(source_path.mtime())
     try:
         fp = open(pyc, "wb")
     except IOError:
@@ -210,7 +227,9 @@
         return False
     try:
         fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
+        mtime = int(source_stat.mtime)
+        size = source_stat.size & 0xFFFFFFFF
+        fp.write(struct.pack("<ll", mtime, size))
         marshal.dump(co, fp)
     finally:
         fp.close()
@@ -225,9 +244,10 @@
 def _rewrite_test(state, fn):
     """Try to read and rewrite *fn* and return the code object."""
     try:
+        stat = fn.stat()
         source = fn.read("rb")
     except EnvironmentError:
-        return None
+        return None, None
     if ASCII_IS_DEFAULT_ENCODING:
         # ASCII is the default encoding in Python 2. Without a coding
         # declaration, Python 2 will complain about any bytes in the file
@@ -246,14 +266,15 @@
             cookie_re.match(source[0:end1]) is None and
             cookie_re.match(source[end1 + 1:end2]) is None):
             if hasattr(state, "_indecode"):
-                return None  # encodings imported us again, we don't rewrite
+                # encodings imported us again, so don't rewrite.
+                return None, None
             state._indecode = True
             try:
                 try:
                     source.decode("ascii")
                 except UnicodeDecodeError:
                     # Let it fail in real import.
-                    return None
+                    return None, None
             finally:
                 del state._indecode
     # On Python versions which are not 2.7 and less than or equal to 3.1, the
@@ -265,7 +286,7 @@
     except SyntaxError:
         # Let this pop up again in the real import.
         state.trace("failed to parse: %r" % (fn,))
-        return None
+        return None, None
     rewrite_asserts(tree)
     try:
         co = compile(tree, fn.strpath, "exec")
@@ -273,23 +294,23 @@
         # It's possible that this error is from some bug in the
         # assertion rewriting, but I don't know of a fast way to tell.
         state.trace("failed to compile: %r" % (fn,))
-        return None
-    return co
+        return None, None
+    return stat, co
 
-def _make_rewritten_pyc(state, fn, pyc, co):
+def _make_rewritten_pyc(state, source_stat, pyc, co):
     """Try to dump rewritten code to *pyc*."""
     if sys.platform.startswith("win"):
         # Windows grants exclusive access to open files and doesn't have atomic
         # rename, so just write into the final file.
-        _write_pyc(state, co, fn, pyc)
+        _write_pyc(state, co, source_stat, pyc)
     else:
         # When not on windows, assume rename is atomic. Dump the code object
         # into a file specific to this process and atomically replace it.
         proc_pyc = pyc + "." + str(os.getpid())
-        if _write_pyc(state, co, fn, proc_pyc):
+        if _write_pyc(state, co, source_stat, proc_pyc):
             os.rename(proc_pyc, pyc)
 
-def _read_pyc(source, pyc):
+def _read_pyc(source, pyc, trace=lambda x: None):
     """Possibly read a pytest pyc containing rewritten code.
 
     Return rewritten code if successful or None if not.
@@ -298,23 +319,28 @@
         fp = open(pyc, "rb")
     except IOError:
         return None
-    try:
+    with fp:
         try:
             mtime = int(source.mtime())
-            data = fp.read(8)
-        except EnvironmentError:
+            size = source.size()
+            data = fp.read(12)
+        except EnvironmentError as e:
+            trace('_read_pyc(%s): EnvironmentError %s' % (source, e))
             return None
         # Check for invalid or out of date pyc file.
-        if (len(data) != 8 or data[:4] != imp.get_magic() or
-                struct.unpack("<l", data[4:])[0] != mtime):
+        if (len(data) != 12 or data[:4] != imp.get_magic() or
+                struct.unpack("<ll", data[4:]) != (mtime, size)):
+            trace('_read_pyc(%s): invalid or out of date pyc' % source)
             return None
-        co = marshal.load(fp)
+        try:
+            co = marshal.load(fp)
+        except Exception as e:
+            trace('_read_pyc(%s): marshal.load error %s' % (source, e))
+            return None
         if not isinstance(co, types.CodeType):
-            # That's interesting....
+            trace('_read_pyc(%s): not a code object' % source)
             return None
         return co
-    finally:
-        fp.close()
 
 
 def rewrite_asserts(mod):
@@ -322,14 +348,64 @@
     AssertionRewriter().run(mod)
 
 
-_saferepr = py.io.saferepr
+def _saferepr(obj):
+    """Get a safe repr of an object for assertion error messages.
+
+    The assertion formatting (util.format_explanation()) requires
+    newlines to be escaped since they are a special character for it.
+    Normally assertion.util.format_explanation() does this but for a
+    custom repr it is possible to contain one of the special escape
+    sequences, especially '\n{' and '\n}' are likely to be present in
+    JSON reprs.
+
+    """
+    repr = py.io.saferepr(obj)
+    if py.builtin._istext(repr):
+        t = py.builtin.text
+    else:
+        t = py.builtin.bytes
+    return repr.replace(t("\n"), t("\\n"))
+
+
 from _pytest.assertion.util import format_explanation as _format_explanation # noqa
 
+def _format_assertmsg(obj):
+    """Format the custom assertion message given.
+
+    For strings this simply replaces newlines with '\n~' so that
+    util.format_explanation() will preserve them instead of escaping
+    newlines.  For other objects py.io.saferepr() is used first.
+
+    """
+    # reprlib appears to have a bug which means that if a string
+    # contains a newline it gets escaped, however if an object has a
+    # .__repr__() which contains newlines it does not get escaped.
+    # However in either case we want to preserve the newline.
+    if py.builtin._istext(obj) or py.builtin._isbytes(obj):
+        s = obj
+        is_repr = False
+    else:
+        s = py.io.saferepr(obj)
+        is_repr = True
+    if py.builtin._istext(s):
+        t = py.builtin.text
+    else:
+        t = py.builtin.bytes
+    s = s.replace(t("\n"), t("\n~")).replace(t("%"), t("%%"))
+    if is_repr:
+        s = s.replace(t("\\n"), t("\n~"))
+    return s
+
 def _should_repr_global_name(obj):
     return not hasattr(obj, "__name__") and not py.builtin.callable(obj)
 
 def _format_boolop(explanations, is_or):
-    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
+    explanation = "(" + (is_or and " or " or " and ").join(explanations) + ")"
+    if py.builtin._istext(explanation):
+        t = py.builtin.text
+    else:
+        t = py.builtin.bytes
+    return explanation.replace(t('%'), t('%%'))
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -377,6 +453,18 @@
     ast.In: "in",
     ast.NotIn: "not in"
 }
+# Python 3.5+ compatibility
+try:
+    binop_map[ast.MatMult] = "@"
+except AttributeError:
+    pass
+
+# Python 3.4+ compatibility
+if hasattr(ast, "NameConstant"):
+    _NameConstant = ast.NameConstant
+else:
+    def _NameConstant(c):
+        return ast.Name(str(c), ast.Load())
 
 
 def set_location(node, lineno, col_offset):
@@ -393,6 +481,56 @@
 
 
 class AssertionRewriter(ast.NodeVisitor):
+    """Assertion rewriting implementation.
+
+    The main entrypoint is to call .run() with an ast.Module instance,
+    this will then find all the assert statements and re-write them to
+    provide intermediate values and a detailed assertion error.  See
+    http://pybites.blogspot.be/2011/07/behind-scenes-of-pytests-new-assertion.html
+    for an overview of how this works.
+
+    The entry point here is .run() which will iterate over all the
+    statements in an ast.Module and for each ast.Assert statement it
+    finds call .visit() with it.  Then .visit_Assert() takes over and
+    is responsible for creating new ast statements to replace the
+    original assert statement: it re-writes the test of an assertion
+    to provide intermediate values and replace it with an if statement
+    which raises an assertion error with a detailed explanation in
+    case the expression is false.
+
+    For this .visit_Assert() uses the visitor pattern to visit all the
+    AST nodes of the ast.Assert.test field, each visit call returning
+    an AST node and the corresponding explanation string.  During this
+    state is kept in several instance attributes:
+
+    :statements: All the AST statements which will replace the assert
+       statement.
+
+    :variables: This is populated by .variable() with each variable
+       used by the statements so that they can all be set to None at
+       the end of the statements.
+
+    :variable_counter: Counter to create new unique variables needed
+       by statements.  Variables are created using .variable() and
+       have the form of "@py_assert0".
+
+    :on_failure: The AST statements which will be executed if the
+       assertion test fails.  This is the code which will construct
+       the failure message and raises the AssertionError.
+
+    :explanation_specifiers: A dict filled by .explanation_param()
+       with %-formatting placeholders and their corresponding
+       expressions to use in the building of an assertion message.
+       This is used by .pop_format_context() to build a message.
+
+    :stack: A stack of the explanation_specifiers dicts maintained by
+       .push_format_context() and .pop_format_context() which allows
+       to build another %-formatted string while already building one.
+
+    This state is reset on every new assert statement visited and used
+    by the other visitors.
+
+    """
 
     def run(self, mod):
         """Find all assert statements in *mod* and rewrite them."""
@@ -466,7 +604,7 @@
         """Call a helper in this module."""
         py_name = ast.Name("@pytest_ar", ast.Load())
         attr = ast.Attribute(py_name, "_" + name, ast.Load())
-        return ast.Call(attr, list(args), [], None, None)
+        return ast_Call(attr, list(args), [])
 
     def builtin(self, name):
         """Return the builtin called *name*."""
@@ -474,15 +612,41 @@
         return ast.Attribute(builtin_name, name, ast.Load())
 
     def explanation_param(self, expr):
+        """Return a new named %-formatting placeholder for expr.
+
+        This creates a %-formatting placeholder for expr in the
+        current formatting context, e.g. ``%(py0)s``.  The placeholder
+        and expr are placed in the current format context so that it
+        can be used on the next call to .pop_format_context().
+
+        """
         specifier = "py" + str(next(self.variable_counter))
         self.explanation_specifiers[specifier] = expr
         return "%(" + specifier + ")s"
 
     def push_format_context(self):
+        """Create a new formatting context.
+
+        The format context is used for when an explanation wants to
+        have a variable value formatted in the assertion message.  In
+        this case the value required can be added using
+        .explanation_param().  Finally .pop_format_context() is used
+        to format a string of %-formatted values as added by
+        .explanation_param().
+
+        """
         self.explanation_specifiers = {}
         self.stack.append(self.explanation_specifiers)
 
     def pop_format_context(self, expl_expr):
+        """Format the %-formatted string with current format context.
+
+        The expl_expr should be an ast.Str instance constructed from
+        the %-placeholders created by .explanation_param().  This will
+        add the required code to format said string to .on_failure and
+        return the ast.Name instance of the formatted string.
+
+        """
         current = self.stack.pop()
         if self.stack:
             self.explanation_specifiers = self.stack[-1]
@@ -500,11 +664,15 @@
         return res, self.explanation_param(self.display(res))
 
     def visit_Assert(self, assert_):
-        if assert_.msg:
-            # There's already a message. Don't mess with it.
-            return [assert_]
+        """Return the AST statements to replace the ast.Assert instance.
+
+        This re-writes the test of an assertion to provide
+        intermediate values and replace it with an if statement which
+        raises an assertion error with a detailed explanation in case
+        the expression is false.
+
+        """
         self.statements = []
-        self.cond_chain = ()
         self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
@@ -516,12 +684,17 @@
         body = self.on_failure
         negation = ast.UnaryOp(ast.Not(), top_condition)
         self.statements.append(ast.If(negation, body, []))
-        explanation = "assert " + explanation
-        template = ast.Str(explanation)
+        if assert_.msg:
+            assertmsg = self.helper('format_assertmsg', assert_.msg)
+            explanation = "\n>assert " + explanation
+        else:
+            assertmsg = ast.Str("")
+            explanation = "assert " + explanation
+        template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
         msg = self.pop_format_context(template)
         fmt = self.helper("format_explanation", msg)
         err_name = ast.Name("AssertionError", ast.Load())
-        exc = ast.Call(err_name, [fmt], [], None, None)
+        exc = ast_Call(err_name, [fmt], [])
         if sys.version_info[0] >= 3:
             raise_ = ast.Raise(exc, None)
         else:
@@ -531,7 +704,7 @@
         if self.variables:
             variables = [ast.Name(name, ast.Store())
                          for name in self.variables]
-            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            clear = ast.Assign(variables, _NameConstant(None))
             self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
@@ -541,7 +714,7 @@
     def visit_Name(self, name):
         # Display the repr of the name if it's a local variable or
         # _should_repr_global_name() thinks it's acceptable.
-        locs = ast.Call(self.builtin("locals"), [], [], None, None)
+        locs = ast_Call(self.builtin("locals"), [], [])
         inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs])
         dorepr = self.helper("should_repr_global_name", name)
         test = ast.BoolOp(ast.Or(), [inlocs, dorepr])
@@ -568,7 +741,7 @@
             res, expl = self.visit(v)
             body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
             expl_format = self.pop_format_context(ast.Str(expl))
-            call = ast.Call(app, [expl_format], [], None, None)
+            call = ast_Call(app, [expl_format], [])
             self.on_failure.append(ast.Expr(call))
             if i < levels:
                 cond = res
@@ -597,7 +770,42 @@
         res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
         return res, explanation
 
-    def visit_Call(self, call):
+    def visit_Call_35(self, call):
+        """
+        visit `ast.Call` nodes on Python3.5 and after
+        """
+        new_func, func_expl = self.visit(call.func)
+        arg_expls = []
+        new_args = []
+        new_kwargs = []
+        for arg in call.args:
+            res, expl = self.visit(arg)
+            arg_expls.append(expl)
+            new_args.append(res)
+        for keyword in call.keywords:
+            res, expl = self.visit(keyword.value)
+            new_kwargs.append(ast.keyword(keyword.arg, res))
+            if keyword.arg:
+                arg_expls.append(keyword.arg + "=" + expl)
+            else: ## **args have `arg` keywords with an .arg of None
+                arg_expls.append("**" + expl)
+
+        expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
+        new_call = ast.Call(new_func, new_args, new_kwargs)
+        res = self.assign(new_call)
+        res_expl = self.explanation_param(self.display(res))
+        outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
+        return res, outer_expl
+
+    def visit_Starred(self, starred):
+        # From Python 3.5, a Starred node can appear in a function call
+        res, expl = self.visit(starred.value)
+        return starred, '*' + expl
+
+    def visit_Call_legacy(self, call):
+        """
+        visit `ast.Call nodes on 3.4 and below`
+        """

From pypy.commits at gmail.com  Tue Nov 15 14:45:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 11:45:19 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: Add missing _pytest files
Message-ID: <582b65cf.272cc20a.38784.d3a7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88397:abcd2a32778c
Date: 2016-11-15 19:44 +0000
http://bitbucket.org/pypy/pypy/changeset/abcd2a32778c/

Log:	Add missing _pytest files

diff too long, truncating to 2000 out of 2510 lines

diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/cacheprovider.py b/_pytest/cacheprovider.py
new file mode 100755
--- /dev/null
+++ b/_pytest/cacheprovider.py
@@ -0,0 +1,245 @@
+"""
+merged implementation of the cache provider
+
+the name cache was not choosen to ensure pluggy automatically
+ignores the external pytest-cache
+"""
+
+import py
+import pytest
+import json
+from os.path import sep as _sep, altsep as _altsep
+
+
+class Cache(object):
+    def __init__(self, config):
+        self.config = config
+        self._cachedir = config.rootdir.join(".cache")
+        self.trace = config.trace.root.get("cache")
+        if config.getvalue("cacheclear"):
+            self.trace("clearing cachedir")
+            if self._cachedir.check():
+                self._cachedir.remove()
+            self._cachedir.mkdir()
+
+    def makedir(self, name):
+        """ return a directory path object with the given name.  If the
+        directory does not yet exist, it will be created.  You can use it
+        to manage files likes e. g. store/retrieve database
+        dumps across test sessions.
+
+        :param name: must be a string not containing a ``/`` separator.
+             Make sure the name contains your plugin or application
+             identifiers to prevent clashes with other cache users.
+        """
+        if _sep in name or _altsep is not None and _altsep in name:
+            raise ValueError("name is not allowed to contain path separators")
+        return self._cachedir.ensure_dir("d", name)
+
+    def _getvaluepath(self, key):
+        return self._cachedir.join('v', *key.split('/'))
+
+    def get(self, key, default):
+        """ return cached value for the given key.  If no value
+        was yet cached or the value cannot be read, the specified
+        default is returned.
+
+        :param key: must be a ``/`` separated value. Usually the first
+             name is the name of your plugin or your application.
+        :param default: must be provided in case of a cache-miss or
+             invalid cache values.
+
+        """
+        path = self._getvaluepath(key)
+        if path.check():
+            try:
+                with path.open("r") as f:
+                    return json.load(f)
+            except ValueError:
+                self.trace("cache-invalid at %s" % (path,))
+        return default
+
+    def set(self, key, value):
+        """ save value for the given key.
+
+        :param key: must be a ``/`` separated value. Usually the first
+             name is the name of your plugin or your application.
+        :param value: must be of any combination of basic
+               python types, including nested types
+               like e. g. lists of dictionaries.
+        """
+        path = self._getvaluepath(key)
+        try:
+            path.dirpath().ensure_dir()
+        except (py.error.EEXIST, py.error.EACCES):
+            self.config.warn(
+                code='I9', message='could not create cache path %s' % (path,)
+            )
+            return
+        try:
+            f = path.open('w')
+        except py.error.ENOTDIR:
+            self.config.warn(
+                code='I9', message='cache could not write path %s' % (path,))
+        else:
+            with f:
+                self.trace("cache-write %s: %r" % (key, value,))
+                json.dump(value, f, indent=2, sort_keys=True)
+
+
+class LFPlugin:
+    """ Plugin which implements the --lf (run last-failing) option """
+    def __init__(self, config):
+        self.config = config
+        active_keys = 'lf', 'failedfirst'
+        self.active = any(config.getvalue(key) for key in active_keys)
+        if self.active:
+            self.lastfailed = config.cache.get("cache/lastfailed", {})
+        else:
+            self.lastfailed = {}
+
+    def pytest_report_header(self):
+        if self.active:
+            if not self.lastfailed:
+                mode = "run all (no recorded failures)"
+            else:
+                mode = "rerun last %d failures%s" % (
+                    len(self.lastfailed),
+                    " first" if self.config.getvalue("failedfirst") else "")
+            return "run-last-failure: %s" % mode
+
+    def pytest_runtest_logreport(self, report):
+        if report.failed and "xfail" not in report.keywords:
+            self.lastfailed[report.nodeid] = True
+        elif not report.failed:
+            if report.when == "call":
+                self.lastfailed.pop(report.nodeid, None)
+
+    def pytest_collectreport(self, report):
+        passed = report.outcome in ('passed', 'skipped')
+        if passed:
+            if report.nodeid in self.lastfailed:
+                self.lastfailed.pop(report.nodeid)
+                self.lastfailed.update(
+                    (item.nodeid, True)
+                    for item in report.result)
+        else:
+            self.lastfailed[report.nodeid] = True
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        if self.active and self.lastfailed:
+            previously_failed = []
+            previously_passed = []
+            for item in items:
+                if item.nodeid in self.lastfailed:
+                    previously_failed.append(item)
+                else:
+                    previously_passed.append(item)
+            if not previously_failed and previously_passed:
+                # running a subset of all tests with recorded failures outside
+                # of the set of tests currently executing
+                pass
+            elif self.config.getvalue("failedfirst"):
+                items[:] = previously_failed + previously_passed
+            else:
+                items[:] = previously_failed
+                config.hook.pytest_deselected(items=previously_passed)
+
+    def pytest_sessionfinish(self, session):
+        config = self.config
+        if config.getvalue("cacheshow") or hasattr(config, "slaveinput"):
+            return
+        prev_failed = config.cache.get("cache/lastfailed", None) is not None
+        if (session.testscollected and prev_failed) or self.lastfailed:
+            config.cache.set("cache/lastfailed", self.lastfailed)
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("general")
+    group.addoption(
+        '--lf', '--last-failed', action='store_true', dest="lf",
+        help="rerun only the tests that failed "
+             "at the last run (or all if none failed)")
+    group.addoption(
+        '--ff', '--failed-first', action='store_true', dest="failedfirst",
+        help="run all tests but run the last failures first.  "
+             "This may re-order tests and thus lead to "
+             "repeated fixture setup/teardown")
+    group.addoption(
+        '--cache-show', action='store_true', dest="cacheshow",
+        help="show cache contents, don't perform collection or tests")
+    group.addoption(
+        '--cache-clear', action='store_true', dest="cacheclear",
+        help="remove all cache contents at start of test run.")
+
+
+def pytest_cmdline_main(config):
+    if config.option.cacheshow:
+        from _pytest.main import wrap_session
+        return wrap_session(config, cacheshow)
+
+
+
+ at pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    config.cache = Cache(config)
+    config.pluginmanager.register(LFPlugin(config), "lfplugin")
+
+
+ at pytest.fixture
+def cache(request):
+    """
+    Return a cache object that can persist state between testing sessions.
+
+    cache.get(key, default)
+    cache.set(key, value)
+
+    Keys must be a ``/`` separated value, where the first part is usually the
+    name of your plugin or application to avoid clashes with other cache users.
+
+    Values can be any object handled by the json stdlib module.
+    """
+    return request.config.cache
+
+
+def pytest_report_header(config):
+    if config.option.verbose:
+        relpath = py.path.local().bestrelpath(config.cache._cachedir)
+        return "cachedir: %s" % relpath
+
+
+def cacheshow(config, session):
+    from pprint import pprint
+    tw = py.io.TerminalWriter()
+    tw.line("cachedir: " + str(config.cache._cachedir))
+    if not config.cache._cachedir.check():
+        tw.line("cache is empty")
+        return 0
+    dummy = object()
+    basedir = config.cache._cachedir
+    vdir = basedir.join("v")
+    tw.sep("-", "cache values")
+    for valpath in vdir.visit(lambda x: x.isfile()):
+        key = valpath.relto(vdir).replace(valpath.sep, "/")
+        val = config.cache.get(key, dummy)
+        if val is dummy:
+            tw.line("%s contains unreadable content, "
+                  "will be ignored" % key)
+        else:
+            tw.line("%s contains:" % key)
+            stream = py.io.TextIO()
+            pprint(val, stream=stream)
+            for line in stream.getvalue().splitlines():
+                tw.line("  " + line)
+
+    ddir = basedir.join("d")
+    if ddir.isdir() and ddir.listdir():
+        tw.sep("-", "cache directories")
+        for p in basedir.join("d").visit():
+            #if p.check(dir=1):
+            #    print("%s/" % p.relto(basedir))
+            if p.isfile():
+                key = p.relto(basedir)
+                tw.line("%s is a file of length %d" % (
+                        key, p.size()))
+    return 0
diff --git a/_pytest/vendored_packages/README.md b/_pytest/vendored_packages/README.md
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/README.md
@@ -0,0 +1,13 @@
+This directory vendors the `pluggy` module.
+
+For a more detailed discussion for the reasons to vendoring this 
+package, please see [this issue](https://github.com/pytest-dev/pytest/issues/944).
+
+To update the current version, execute:
+
+```
+$ pip install -U pluggy==<version> --no-compile --target=_pytest/vendored_packages
+```
+
+And commit the modified files. The `pluggy-<version>.dist-info` directory 
+created by `pip` should be ignored.
diff --git a/_pytest/vendored_packages/__init__.py b/_pytest/vendored_packages/__init__.py
new file mode 100644
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/DESCRIPTION.rst b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/DESCRIPTION.rst
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/DESCRIPTION.rst
@@ -0,0 +1,10 @@
+Plugin registration and hook calling for Python
+===============================================
+
+This is the plugin manager as used by pytest but stripped
+of pytest specific details.
+
+During the 0.x series this plugin does not have much documentation
+except extensive docstrings in the pluggy.py module.
+
+
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/METADATA b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/METADATA
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/METADATA
@@ -0,0 +1,39 @@
+Metadata-Version: 2.0
+Name: pluggy
+Version: 0.3.1
+Summary: plugin and hook calling mechanisms for python
+Home-page: UNKNOWN
+Author: Holger Krekel
+Author-email: holger at merlinux.eu
+License: MIT license
+Platform: unix
+Platform: linux
+Platform: osx
+Platform: win32
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+
+Plugin registration and hook calling for Python
+===============================================
+
+This is the plugin manager as used by pytest but stripped
+of pytest specific details.
+
+During the 0.x series this plugin does not have much documentation
+except extensive docstrings in the pluggy.py module.
+
+
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/RECORD b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/RECORD
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/RECORD
@@ -0,0 +1,8 @@
+pluggy.py,sha256=v_RfWzyW6DPU1cJu_EFoL_OHq3t13qloVdR6UaMCXQA,29862
+pluggy-0.3.1.dist-info/top_level.txt,sha256=xKSCRhai-v9MckvMuWqNz16c1tbsmOggoMSwTgcpYHE,7
+pluggy-0.3.1.dist-info/pbr.json,sha256=xX3s6__wOcAyF-AZJX1sdZyW6PUXT-FkfBlM69EEUCg,47
+pluggy-0.3.1.dist-info/RECORD,,
+pluggy-0.3.1.dist-info/metadata.json,sha256=nLKltOT78dMV-00uXD6Aeemp4xNsz2q59j6ORSDeLjw,1027
+pluggy-0.3.1.dist-info/METADATA,sha256=1b85Ho2u4iK30M099k7axMzcDDhLcIMb-A82JUJZnSo,1334
+pluggy-0.3.1.dist-info/WHEEL,sha256=AvR0WeTpDaxT645bl5FQxUK6NPsTls2ttpcGJg3j1Xg,110
+pluggy-0.3.1.dist-info/DESCRIPTION.rst,sha256=P5Akh1EdIBR6CeqtV2P8ZwpGSpZiTKPw0NyS7jEiD-g,306
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/WHEEL b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/WHEEL
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.24.0)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/metadata.json b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/metadata.json
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/metadata.json
@@ -0,0 +1,1 @@
+{"license": "MIT license", "name": "pluggy", "metadata_version": "2.0", "generator": "bdist_wheel (0.24.0)", "summary": "plugin and hook calling mechanisms for python", "platform": "unix", "version": "0.3.1", "extensions": {"python.details": {"document_names": {"description": "DESCRIPTION.rst"}, "contacts": [{"role": "author", "email": "holger at merlinux.eu", "name": "Holger Krekel"}]}}, "classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Topic :: Software Development :: Testing", "Topic :: Software Development :: Libraries", "Topic :: Utilities", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5"]}
\ No newline at end of file
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/pbr.json b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/pbr.json
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/pbr.json
@@ -0,0 +1,1 @@
+{"is_release": false, "git_version": "7d4c9cd"}
\ No newline at end of file
diff --git a/_pytest/vendored_packages/pluggy-0.3.1.dist-info/top_level.txt b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/top_level.txt
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy-0.3.1.dist-info/top_level.txt
@@ -0,0 +1,1 @@
+pluggy
diff --git a/_pytest/vendored_packages/pluggy.py b/_pytest/vendored_packages/pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/vendored_packages/pluggy.py
@@ -0,0 +1,777 @@
+"""
+PluginManager, basic initialization and tracing.
+
+pluggy is the cristallized core of plugin management as used
+by some 150 plugins for pytest.
+
+Pluggy uses semantic versioning. Breaking changes are only foreseen for
+Major releases (incremented X in "X.Y.Z").  If you want to use pluggy in
+your project you should thus use a dependency restriction like
+"pluggy>=0.1.0,<1.0" to avoid surprises.
+
+pluggy is concerned with hook specification, hook implementations and hook
+calling.  For any given hook specification a hook call invokes up to N implementations.
+A hook implementation can influence its position and type of execution:
+if attributed "tryfirst" or "trylast" it will be tried to execute
+first or last.  However, if attributed "hookwrapper" an implementation
+can wrap all calls to non-hookwrapper implementations.  A hookwrapper
+can thus execute some code ahead and after the execution of other hooks.
+
+Hook specification is done by way of a regular python function where
+both the function name and the names of all its arguments are significant.
+Each hook implementation function is verified against the original specification
+function, including the names of all its arguments.  To allow for hook specifications
+to evolve over the livetime of a project, hook implementations can
+accept less arguments.  One can thus add new arguments and semantics to
+a hook specification by adding another argument typically without breaking
+existing hook implementations.
+
+The chosen approach is meant to let a hook designer think carefuly about
+which objects are needed by an extension writer.  By contrast, subclass-based
+extension mechanisms often expose a lot more state and behaviour than needed,
+thus restricting future developments.
+
+Pluggy currently consists of functionality for:
+
+- a way to register new hook specifications.  Without a hook
+  specification no hook calling can be performed.
+
+- a registry of plugins which contain hook implementation functions.  It
+  is possible to register plugins for which a hook specification is not yet
+  known and validate all hooks when the system is in a more referentially
+  consistent state.  Setting an "optionalhook" attribution to a hook
+  implementation will avoid PluginValidationError's if a specification
+  is missing.  This allows to have optional integration between plugins.
+
+- a "hook" relay object from which you can launch 1:N calls to
+  registered hook implementation functions
+
+- a mechanism for ordering hook implementation functions
+
+- mechanisms for two different type of 1:N calls: "firstresult" for when
+  the call should stop when the first implementation returns a non-None result.
+  And the other (default) way of guaranteeing that all hook implementations
+  will be called and their non-None result collected.
+
+- mechanisms for "historic" extension points such that all newly
+  registered functions will receive all hook calls that happened
+  before their registration.
+
+- a mechanism for discovering plugin objects which are based on
+  setuptools based entry points.
+
+- a simple tracing mechanism, including tracing of plugin calls and
+  their arguments.
+
+"""
+import sys
+import inspect
+
+__version__ = '0.3.1'
+__all__ = ["PluginManager", "PluginValidationError",
+           "HookspecMarker", "HookimplMarker"]
+
+_py3 = sys.version_info > (3, 0)
+
+
+class HookspecMarker:
+    """ Decorator helper class for marking functions as hook specifications.
+
+    You can instantiate it with a project_name to get a decorator.
+    Calling PluginManager.add_hookspecs later will discover all marked functions
+    if the PluginManager uses the same project_name.
+    """
+
+    def __init__(self, project_name):
+        self.project_name = project_name
+
+    def __call__(self, function=None, firstresult=False, historic=False):
+        """ if passed a function, directly sets attributes on the function
+        which will make it discoverable to add_hookspecs().  If passed no
+        function, returns a decorator which can be applied to a function
+        later using the attributes supplied.
+
+        If firstresult is True the 1:N hook call (N being the number of registered
+        hook implementation functions) will stop at I<=N when the I'th function
+        returns a non-None result.
+
+        If historic is True calls to a hook will be memorized and replayed
+        on later registered plugins.
+
+        """
+        def setattr_hookspec_opts(func):
+            if historic and firstresult:
+                raise ValueError("cannot have a historic firstresult hook")
+            setattr(func, self.project_name + "_spec",
+                   dict(firstresult=firstresult, historic=historic))
+            return func
+
+        if function is not None:
+            return setattr_hookspec_opts(function)
+        else:
+            return setattr_hookspec_opts
+
+
+class HookimplMarker:
+    """ Decorator helper class for marking functions as hook implementations.
+
+    You can instantiate with a project_name to get a decorator.
+    Calling PluginManager.register later will discover all marked functions
+    if the PluginManager uses the same project_name.
+    """
+    def __init__(self, project_name):
+        self.project_name = project_name
+
+    def __call__(self, function=None, hookwrapper=False, optionalhook=False,
+                 tryfirst=False, trylast=False):
+
+        """ if passed a function, directly sets attributes on the function
+        which will make it discoverable to register().  If passed no function,
+        returns a decorator which can be applied to a function later using
+        the attributes supplied.
+
+        If optionalhook is True a missing matching hook specification will not result
+        in an error (by default it is an error if no matching spec is found).
+
+        If tryfirst is True this hook implementation will run as early as possible
+        in the chain of N hook implementations for a specfication.
+
+        If trylast is True this hook implementation will run as late as possible
+        in the chain of N hook implementations.
+
+        If hookwrapper is True the hook implementations needs to execute exactly
+        one "yield".  The code before the yield is run early before any non-hookwrapper
+        function is run.  The code after the yield is run after all non-hookwrapper
+        function have run.  The yield receives an ``_CallOutcome`` object representing
+        the exception or result outcome of the inner calls (including other hookwrapper
+        calls).
+
+        """
+        def setattr_hookimpl_opts(func):
+            setattr(func, self.project_name + "_impl",
+                   dict(hookwrapper=hookwrapper, optionalhook=optionalhook,
+                        tryfirst=tryfirst, trylast=trylast))
+            return func
+
+        if function is None:
+            return setattr_hookimpl_opts
+        else:
+            return setattr_hookimpl_opts(function)
+
+
+def normalize_hookimpl_opts(opts):
+    opts.setdefault("tryfirst", False)
+    opts.setdefault("trylast", False)
+    opts.setdefault("hookwrapper", False)
+    opts.setdefault("optionalhook", False)
+
+
+class _TagTracer:
+    def __init__(self):
+        self._tag2proc = {}
+        self.writer = None
+        self.indent = 0
+
+    def get(self, name):
+        return _TagTracerSub(self, (name,))
+
+    def format_message(self, tags, args):
+        if isinstance(args[-1], dict):
+            extra = args[-1]
+            args = args[:-1]
+        else:
+            extra = {}
+
+        content = " ".join(map(str, args))
+        indent = "  " * self.indent
+
+        lines = [
+            "%s%s [%s]\n" % (indent, content, ":".join(tags))
+        ]
+
+        for name, value in extra.items():
+            lines.append("%s    %s: %s\n" % (indent, name, value))
+        return lines
+
+    def processmessage(self, tags, args):
+        if self.writer is not None and args:
+            lines = self.format_message(tags, args)
+            self.writer(''.join(lines))
+        try:
+            self._tag2proc[tags](tags, args)
+        except KeyError:
+            pass
+
+    def setwriter(self, writer):
+        self.writer = writer
+
+    def setprocessor(self, tags, processor):
+        if isinstance(tags, str):
+            tags = tuple(tags.split(":"))
+        else:
+            assert isinstance(tags, tuple)
+        self._tag2proc[tags] = processor
+
+
+class _TagTracerSub:
+    def __init__(self, root, tags):
+        self.root = root
+        self.tags = tags
+
+    def __call__(self, *args):
+        self.root.processmessage(self.tags, args)
+
+    def setmyprocessor(self, processor):
+        self.root.setprocessor(self.tags, processor)
+
+    def get(self, name):
+        return self.__class__(self.root, self.tags + (name,))
+
+
+def _raise_wrapfail(wrap_controller, msg):
+    co = wrap_controller.gi_code
+    raise RuntimeError("wrap_controller at %r %s:%d %s" %
+                   (co.co_name, co.co_filename, co.co_firstlineno, msg))
+
+
+def _wrapped_call(wrap_controller, func):
+    """ Wrap calling to a function with a generator which needs to yield
+    exactly once.  The yield point will trigger calling the wrapped function
+    and return its _CallOutcome to the yield point.  The generator then needs
+    to finish (raise StopIteration) in order for the wrapped call to complete.
+    """
+    try:
+        next(wrap_controller)   # first yield
+    except StopIteration:
+        _raise_wrapfail(wrap_controller, "did not yield")
+    call_outcome = _CallOutcome(func)
+    try:
+        wrap_controller.send(call_outcome)
+        _raise_wrapfail(wrap_controller, "has second yield")
+    except StopIteration:
+        pass
+    return call_outcome.get_result()
+
+
+class _CallOutcome:
+    """ Outcome of a function call, either an exception or a proper result.
+    Calling the ``get_result`` method will return the result or reraise
+    the exception raised when the function was called. """
+    excinfo = None
+
+    def __init__(self, func):
+        try:
+            self.result = func()
+        except BaseException:
+            self.excinfo = sys.exc_info()
+

From pypy.commits at gmail.com  Tue Nov 15 17:08:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 15 Nov 2016 14:08:29 -0800 (PST)
Subject: [pypy-commit] pypy default: write at yet another place this F.A.Q.
Message-ID: <582b875d.e626c20a.c4c66.3ead@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88398:416d9a87c548
Date: 2016-11-15 23:07 +0100
http://bitbucket.org/pypy/pypy/changeset/416d9a87c548/

Log:	write at yet another place this F.A.Q.

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -90,6 +90,10 @@
 Do CPython Extension modules work with PyPy?
 --------------------------------------------
 
+**First note that some Linux distributions (e.g. Ubuntu, Debian) split
+PyPy into several packages.  If you installed a package called "pypy",
+then you may also need to install "pypy-dev" for the following to work.**
+
 We have experimental support for CPython extension modules, so
 they run with minor changes.  This has been a part of PyPy since
 the 1.4 release, but support is still in beta phase.  CPython

From pypy.commits at gmail.com  Tue Nov 15 18:27:06 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 15:27:06 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: Don't interpret exitcode 5 (no
 tests were run) as a failure
Message-ID: <582b99ca.c4251c0a.ded54.3bdc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88399:679a4be18d09
Date: 2016-11-15 23:26 +0000
http://bitbucket.org/pypy/pypy/changeset/679a4be18d09/

Log:	Don't interpret exitcode 5 (no tests were run) as a failure

diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -127,9 +127,9 @@
         runfunc = dry_run
     else:
         runfunc = run
-    
+
     exitcode = runfunc(args, cwd, out, timeout=timeout)
-    
+
     return exitcode
 
 def should_report_failure(logdata):
@@ -147,7 +147,7 @@
 
 def interpret_exitcode(exitcode, test, logdata=""):
     extralog = ""
-    if exitcode:
+    if exitcode not in (0, 5):
         failure = True
         if exitcode != 1 or should_report_failure(logdata):
             if exitcode > 0:
@@ -268,14 +268,14 @@
             out.write("++ %s starting %s [%d started in total]\n" % (now, res[1],
                                                                   started))
             continue
-        
+
         testname, somefailed, logdata, output = res[1:]
         done += 1
         failure = failure or somefailed
 
         heading = "__ %s [%d done in total, somefailed=%s] " % (
             testname, done, somefailed)
-        
+
         out.write(heading + (79-len(heading))*'_'+'\n')
 
         out.write(output)
@@ -299,7 +299,7 @@
     parallel_runs = 1
     timeout = None
     cherrypick = None
-    
+
     def __init__(self, root):
         self.root = root
         self.self = self
@@ -372,7 +372,7 @@
     parser.add_option("--timeout", dest="timeout", default=None,
                       type="int",
                       help="timeout in secs for test processes")
-        
+
     opts, args = parser.parse_args(args)
 
     if opts.logfile is None:
@@ -417,7 +417,7 @@
     if run_param.dry_run:
         print >>out, '\n'.join([str((k, getattr(run_param, k))) \
                         for k in dir(run_param) if k[:2] != '__'])
-    
+
     res = execute_tests(run_param, testdirs, logfile, out)
 
     if res:

From pypy.commits at gmail.com  Tue Nov 15 18:54:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 15:54:12 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: Don't use deprecated __multicall__
Message-ID: <582ba024.c1341c0a.d3e82.4821@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88400:2375f918f240
Date: 2016-11-15 23:38 +0000
http://bitbucket.org/pypy/pypy/changeset/2375f918f240/

Log:	Don't use deprecated __multicall__

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -159,7 +159,8 @@
         return space
 
 
-def pytest_runtest_setup(__multicall__, item):
+ at pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
     if isinstance(item, py.test.collect.Function):
         appclass = item.getparent(py.test.Class)
         if appclass is not None:
@@ -172,8 +173,6 @@
                 appclass.obj.space = LazyObjSpaceGetter()
             appclass.obj.runappdirect = option.runappdirect
 
-    __multicall__.execute()
-
 
 def pytest_ignore_collect(path):
     return path.check(link=1)
diff --git a/rpython/conftest.py b/rpython/conftest.py
--- a/rpython/conftest.py
+++ b/rpython/conftest.py
@@ -45,16 +45,6 @@
            help="show the dependencies that have been constructed from a trace")
 
 
-def pytest_pycollect_makeitem(__multicall__,collector, name, obj):
-    res = __multicall__.execute()
-    # work around pytest issue 251
-    import inspect
-    if res is None and inspect.isclass(obj) and \
-            collector.classnamefilter(name):
-        return py.test.collect.Class(name, parent=collector)
-    return res
-
-
 def pytest_addhooks(pluginmanager):
     pluginmanager.register(LeakFinder())
 
@@ -63,21 +53,21 @@
 
     So far, only used by the function lltype.malloc(flavor='raw').
     """
-    def pytest_runtest_setup(self, __multicall__, item):
-        __multicall__.execute()
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_setup(self, item):
         if not isinstance(item, py.test.collect.Function):
             return
         if not getattr(item.obj, 'dont_track_allocations', False):
             leakfinder.start_tracking_allocations()
 
-    def pytest_runtest_call(self, __multicall__, item):
-        __multicall__.execute()
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_call(self, item):
         if not isinstance(item, py.test.collect.Function):
             return
         item._success = True
 
-    def pytest_runtest_teardown(self, __multicall__, item):
-        __multicall__.execute()
+    @pytest.hookimpl(trylast=True)
+    def pytest_runtest_teardown(self, item):
         if not isinstance(item, py.test.collect.Function):
             return
         if (not getattr(item.obj, 'dont_track_allocations', False)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
--- a/testrunner/test/conftest.py
+++ b/testrunner/test/conftest.py
@@ -1,6 +1,8 @@
+import pytest
 
-def pytest_runtest_makereport(__multicall__, item):
-    report = __multicall__.execute()
+ at pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item):
+    report = yield
     if 'out' in item.funcargs:
         report.sections.append(('out', item.funcargs['out'].read()))
     return report

From pypy.commits at gmail.com  Tue Nov 15 18:54:14 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 15:54:14 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: update testrunner tests
Message-ID: <582ba026.83071c0a.b930c.42f6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88401:ef20239bae31
Date: 2016-11-15 23:53 +0000
http://bitbucket.org/pypy/pypy/changeset/ef20239bae31/

Log:	update testrunner tests

diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
--- a/testrunner/test/conftest.py
+++ b/testrunner/test/conftest.py
@@ -2,7 +2,6 @@
 
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item):
-    report = yield
+    report = (yield).result
     if 'out' in item.funcargs:
         report.sections.append(('out', item.funcargs['out'].read()))
-    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -84,14 +84,14 @@
 
     def test_timeout_syscall(self, out):
         res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
-        assert res == -999        
+        assert res == -999
 
     def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
                          out, timeout=2)
         assert res == 0
         out = out.read()
-        assert out == "42\n"        
+        assert out == "42\n"
 
 
 class TestExecuteTest(object):
@@ -100,7 +100,7 @@
         cls.real_run = (runner.run,)
         cls.called = []
         cls.exitcode = [0]
-        
+
         def fake_run(args, cwd, out, timeout):
             cls.called = (args, cwd, out, timeout)
             return cls.exitcode[0]
@@ -123,7 +123,7 @@
 
                     'test_one']
 
-        assert self.called == (expected, '/wd', 'out', 'secs')        
+        assert self.called == (expected, '/wd', 'out', 'secs')
         assert res == 0
 
     def test_explicit_win32(self):
@@ -141,7 +141,7 @@
                     #'--junitxml=LOGFILE.junit',
                     'test_one']
         assert self.called[0] == expected
-        assert self.called == (expected, '/wd', 'out', 'secs')        
+        assert self.called == (expected, '/wd', 'out', 'secs')
         assert res == 0
 
     def test_error(self):
@@ -193,14 +193,14 @@
         cls.real_invoke_in_thread = (runner.invoke_in_thread,)
         if not cls.with_thread:
             runner.invoke_in_thread = lambda func, args: func(*args)
-        
+
         cls.udir = py.path.local.make_numbered_dir(prefix='usession-runner-',
                                               keep=3)
         cls.manydir = cls.udir.join('many').ensure(dir=1)
 
         cls.udir.join("conftest.py").write("pytest_plugins = 'resultlog'\n")
 
-        def fill_test_dir(test_dir, fromdir='normal'):       
+        def fill_test_dir(test_dir, fromdir='normal'):
             for p in py.path.local(__file__).dirpath(
                 'examples', fromdir).listdir("*.py"):
                 p.copy(test_dir.join('test_'+p.basename))
@@ -210,7 +210,7 @@
         cls.one_test_dir = cls.manydir.join('one')
 
         fill_test_dir(test_normal_dir0)
-        
+
 
         test_normal_dir1 = cls.manydir.join('two', 'test_normal1').ensure(dir=1)
         test_normal_dir2 = cls.manydir.join('two', 'pkg',
@@ -235,8 +235,8 @@
 
         run_param = runner.RunParam(self.one_test_dir)
         run_param.test_driver = test_driver
-        run_param.parallel_runs = 3        
-        
+        run_param.parallel_runs = 3
+
         res = runner.execute_tests(run_param, ['test_normal'], log, out)
 
         assert res
@@ -248,10 +248,10 @@
 
         log = log.getvalue()
         assert '\r\n' not in log
-        assert '\n' in log        
+        assert '\n' in log
         log_lines = log.splitlines()
 
-        assert ". test_normal/test_example.py::test_one" in log_lines
+        assert ". test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:
@@ -273,7 +273,7 @@
         run_param.test_driver = test_driver
         run_param.parallel_runs = 3
         run_param.dry_run = True
-        
+
         res = runner.execute_tests(run_param, ['test_normal'], log, out)
 
         assert not res
@@ -282,11 +282,11 @@
 
         out_lines = out.getvalue().splitlines()
 
-        assert len(out_lines) == 5
+        assert len(out_lines) == 7
 
-        assert out_lines[2].startswith("++ starting")
-        assert out_lines[4].startswith("run [")
-        for line in out_lines[2:]:
+        assert 'starting' in out_lines[4]
+        assert out_lines[6].startswith("run [")
+        for line in out_lines[4:]:
             assert "test_normal" in line
 
     def test_many_dirs(self):
@@ -307,7 +307,7 @@
         testdirs = []
         run_param.collect_testdirs(testdirs)
         alltestdirs = testdirs[:]
-        
+
         res = runner.execute_tests(run_param, testdirs, log, out)
 
         assert res
@@ -366,7 +366,7 @@
 
     def test_run_bad_get_test_driver(self):
         test_driver = [pytest_script]
-        
+
         log = cStringIO.StringIO()
         out = cStringIO.StringIO()
 
@@ -399,7 +399,7 @@
             real_collect_one_testdir(testdirs, reldir, tests)
 
         run_param.collect_one_testdir = witness_collect_one_testdir
-        
+
         run_param.collect_testdirs(res)
 
         assert res == ['test_normal']
@@ -412,10 +412,10 @@
         run_param.collect_one_testdir = real_collect_one_testdir
         res = []
         run_param = runner.RunParam(self.two_test_dir)
-        
+
         run_param.collect_testdirs(res)
 
-        assert sorted(res) == ['pkg/test_normal2', 'test_normal1']        
+        assert sorted(res) == ['pkg/test_normal2', 'test_normal1']
 
 
 class TestRunner(RunnerTests):

From pypy.commits at gmail.com  Tue Nov 15 22:11:06 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 15 Nov 2016 19:11:06 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: Fix lib-python conftest.py
Message-ID: <582bce4a.88711c0a.80263.6c94@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88402:3d4564d1d24c
Date: 2016-11-16 03:10 +0000
http://bitbucket.org/pypy/pypy/changeset/3d4564d1d24c/

Log:	Fix lib-python conftest.py

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -5,9 +5,10 @@
 
 """
 import py
+import pytest
 import sys
+import re
 import pypy
-import re
 from pypy.interpreter.gateway import ApplevelClass
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.module import Module as PyPyModule
@@ -517,16 +518,16 @@
     for x in testmap:
         cache[x.basename] = x
 
-def pytest_collect_file(path, parent, __multicall__):
-    # don't collect files except through this hook
-    # implemented by clearing the list of to-be-called
-    # remaining hook methods
-    __multicall__.methods[:] = []
-    regrtest = parent.config._basename2spec.get(path.basename, None)
-    if regrtest is None:
-        return
-    if path.dirpath() != testdir:
-        return
+def pytest_ignore_collect(path, config):
+    if path.isfile():
+        regrtest = config._basename2spec.get(path.basename, None)
+        if regrtest is None or path.dirpath() != testdir:
+            return True
+
+ at pytest.hookimpl(tryfirst=True)
+def pytest_pycollect_makemodule(path, parent):
+    config = parent.config
+    regrtest = config._basename2spec[path.basename]
     return RunFileExternal(path.basename, parent=parent, regrtest=regrtest)
 
 class RunFileExternal(py.test.collect.File):

From pypy.commits at gmail.com  Wed Nov 16 03:50:03 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 00:50:03 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: hack around a missing "self"
 argument in slot_tp_new
Message-ID: <582c1dbb.4dd41c0a.abf42.dccf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88404:48bcbfaa8980
Date: 2016-11-16 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/48bcbfaa8980/

Log:	hack around a missing "self" argument in slot_tp_new

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1003,7 +1003,8 @@
         import datetime
         module = self.import_module(name='foo3')
         module.footype("X", (object,), {})
-        module.datetimetype(1, 1, 1)
+        a = module.datetimetype(1, 1, 1)
+        assert isinstance(a, module.datetimetype)
 
     def test_app_subclass_of_c_type(self):
         import sys
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -309,6 +309,8 @@
                     STRUCT_TYPE = PySequenceMethods
                 elif slot_names[0] == 'c_tp_as_buffer':
                     STRUCT_TYPE = PyBufferProcs
+                elif slot_names[0] == 'c_tp_as_mapping':
+                    STRUCT_TYPE = PyMappingMethods
                 else:
                     raise AssertionError(
                         "Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -30,8 +30,20 @@
 
 @cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None)
 def slot_tp_new(space, w_type, w_args, w_kwds):
-    w_impl = space.getattr(w_type, space.wrap('__new__'))
-    import pdb;pdb.set_trace()
+    # XXX problem - we need to find the actual __new__ function to call.
+    #     but we have no 'self' argument. Theoretically, self will be
+    #     w_type, but if w_type is a subclass of self, and w_type has a
+    #     __new__ function that calls super().__new__, and that call ends
+    #     up here, we will get infinite recursion. Prevent the recursion
+    #     in the simple case (from cython) where w_type is a cpytype, but
+    #     we know (since we are in this function) that self is not a cpytype
+    from pypy.module.cpyext.typeobject import W_PyCTypeObject
+    w_type0 = w_type
+    w_mro = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    while w_type0.is_cpytype():
+        w_type0 = w_mro[1]
+        w_mro = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    w_impl = space.getattr(w_type0, space.wrap('__new__'))
     args = Arguments(space, [w_type],
                      w_stararg=w_args, w_starstararg=w_kwds)
     return space.call_args(w_impl, args)

From pypy.commits at gmail.com  Wed Nov 16 03:50:04 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 00:50:04 -0800 (PST)
Subject: [pypy-commit] pypy default: test, add PyAnySet_Check
Message-ID: <582c1dbc.4dd41c0a.abf42.dcd4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88405:25c0b98bd9df
Date: 2016-11-16 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/25c0b98bd9df/

Log:	test, add PyAnySet_Check

diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -11,6 +11,13 @@
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyAnySet_Check(space, w_obj):
+    """Return true if obj is a set object, a frozenset object, or an
+    instance of a subtype."""
+    return (space.isinstance_w(w_obj, space.gettypefor(W_SetObject)) or
+            space.isinstance_w(w_obj, space.gettypefor(W_FrozensetObject)))
+
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyAnySet_CheckExact(space, w_obj):
     """Return true if obj is a set object or a frozenset object but
     not an instance of a subtype."""
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1476,18 +1476,6 @@
     raise NotImplementedError
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyAnySet_Check(space, p):
-    """Return true if p is a set object, a frozenset object, or an
-    instance of a subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyAnySet_CheckExact(space, p):
-    """Return true if p is a set object or a frozenset object but
-    not an instance of a subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyFrozenSet_CheckExact(space, p):
     """Return true if p is a frozenset object but not an instance of a
     subtype."""
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -41,11 +41,19 @@
         api.PySet_Clear(w_set)
         assert space.len_w(w_set) == 0
 
-    def test_anyset_checkexact(self, space, api):
+    def test_anyset_check(self, space, api):
         w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         w_frozenset = space.newfrozenset([space.wrap(i) for i in [1, 2, 3, 4]])
         assert api.PyAnySet_CheckExact(w_set)
         assert api.PyAnySet_CheckExact(w_frozenset)
+        assert api.PyAnySet_Check(w_set)
+        assert api.PyAnySet_Check(w_frozenset)
+        w_instance = space.appexec([], """():
+            class MySet(set):
+                pass
+            return MySet()
+        """)
+        assert api.PyAnySet_Check(w_instance)
 
 class AppTestSetObject(AppTestCpythonExtensionBase):
     def test_set_macro_cast(self):

From pypy.commits at gmail.com  Wed Nov 16 03:50:01 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 00:50:01 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <582c1db9.06891c0a.12730.e26e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88403:8e7bf0562443
Date: 2016-11-15 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/8e7bf0562443/

Log:	merge default into branch

diff too long, truncating to 2000 out of 16218 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -2,6 +2,7 @@
 from ctypes.test import need_symbol, xfail
 import unittest
 import os
+import sys
 
 import ctypes
 import _ctypes_test
@@ -279,7 +280,6 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xef\xcd\xab\x21')
 
-    @xfail
     @need_symbol('c_uint32')
     def test_uint32_swap_big_endian(self):
         # Issue #23319
@@ -295,5 +295,11 @@
         x.c = 2
         self.assertEqual(b.tostring(), b'\xab\xcd\xef\x12')
 
+    # see issue #1213, on big endian it fails for the little endian case
+    if sys.byteorder == 'little':
+        test_uint32_swap_big_endian = xfail(test_uint32_swap_big_endian)
+    elif sys.byteorder == 'big':
+        test_uint32_swap_little_endian = xfail(test_uint32_swap_little_endian)
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -66,6 +66,7 @@
     g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['OPT'] = "" 
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -26,6 +26,8 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
+IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
+IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -742,15 +744,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
-                         ctx.options)
+        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
+        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
+            default |= ssl.OP_NO_COMPRESSION
+        self.assertEqual(default, ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
-                         ctx.options)
+        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
-            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
-                             ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
+            self.assertEqual(default, ctx.options)
             ctx.options = 0
             self.assertEqual(0, ctx.options)
         else:
@@ -2918,18 +2920,27 @@
                 client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-                stats = server_params_test(client_context, server_context,
-                                           chatty=True, connectionchatty=True)
-
-                msg = "failed trying %s (s) and %s (c).\n" \
-                      "was expecting %s, but got %%s from the %%s" \
-                          % (str(server_protocols), str(client_protocols),
-                             str(expected))
-                client_result = stats['client_alpn_protocol']
-                self.assertEqual(client_result, expected, msg % (client_result, "client"))
-                server_result = stats['server_alpn_protocols'][-1] \
-                    if len(stats['server_alpn_protocols']) else 'nothing'
-                self.assertEqual(server_result, expected, msg % (server_result, "server"))
+                try:
+                    stats = server_params_test(client_context, server_context,
+                                               chatty=True, connectionchatty=True)
+                except ssl.SSLError as e:
+                    stats = e
+
+                if expected is None and IS_OPENSSL_1_1:
+                    # OpenSSL 1.1.0 raises handshake error
+                    self.assertIsInstance(stats, ssl.SSLError)
+                else:
+                    msg = "failed trying %s (s) and %s (c).\n" \
+                          "was expecting %s, but got %%s from the %%s" \
+                              % (str(server_protocols), str(client_protocols),
+                                 str(expected))
+                    client_result = stats['client_alpn_protocol']
+                    self.assertEqual(client_result, expected,
+                                     msg % (client_result, "client"))
+                    server_result = stats['server_alpn_protocols'][-1] \
+                        if len(stats['server_alpn_protocols']) else 'nothing'
+                    self.assertEqual(server_result, expected,
+                                     msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -17,7 +17,7 @@
 from pypy.conftest import option as pypy_option
 
 from pypy.tool.pytest import appsupport
-from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir
+from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir
 from rpython.config.parse import parse_info
 
 pytest_plugins = "resultlog",
@@ -561,7 +561,7 @@
             watchdog_name = 'watchdog_nt.py'
         else:
             watchdog_name = 'watchdog.py'
-        watchdog_script = rpythondir.join('tool', watchdog_name)
+        watchdog_script = pypydir.join('tool', watchdog_name)
 
         regr_script = pypydir.join('tool', 'pytest',
                                    'run-script', 'regrverbose.py')
diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -29,7 +29,7 @@
 class deque(object):
 
     def __new__(cls, iterable=(), *args, **kw):
-        self = super(deque, cls).__new__(cls, *args, **kw)
+        self = super(deque, cls).__new__(cls)
         self.clear()
         return self
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -229,7 +229,7 @@
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
-        self = super(_CData, cls).__new__(cls, *args, **kwds)
+        self = super(_CData, cls).__new__(cls)
         if '_abstract_' in cls.__dict__:
             raise TypeError("abstract class")
         if hasattr(cls, '_ffistruct_'):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.8.4
+Version: 1.9.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.8.4"
-__version_info__ = (1, 8, 4)
+__version__ = "1.9.1"
+__version_info__ = (1, 9, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -141,9 +141,9 @@
 #define _cffi_to_c_char                                                  \
                  ((int(*)(PyObject *))_cffi_exports[9])
 #define _cffi_from_c_pointer                                             \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10])
 #define _cffi_to_c_pointer                                               \
-    ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11])
+    ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11])
 #define _cffi_get_struct_layout                                          \
     not used any more
 #define _cffi_restore_errno                                              \
@@ -153,11 +153,11 @@
 #define _cffi_from_c_char                                                \
     ((PyObject *(*)(char))_cffi_exports[15])
 #define _cffi_from_c_deref                                               \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16])
 #define _cffi_to_c                                                       \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17])
 #define _cffi_from_c_struct                                              \
-    ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18])
+    ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
     ((wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
@@ -167,21 +167,22 @@
 #define _cffi_to_c__Bool                                                 \
     ((_Bool(*)(PyObject *))_cffi_exports[22])
 #define _cffi_prepare_pointer_call_argument                              \
-    ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
+    ((Py_ssize_t(*)(struct _cffi_ctypedescr *,                           \
+                    PyObject *, char **))_cffi_exports[23])
 #define _cffi_convert_array_from_object                                  \
-    ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+    ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24])
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
 #define _CFFI_NUM_EXPORTS 26
 
-typedef struct _ctypedescr CTypeDescrObject;
+struct _cffi_ctypedescr;
 
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 
 #define _cffi_type(index)   (                           \
     assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \
-    (CTypeDescrObject *)_cffi_types[index])
+    (struct _cffi_ctypedescr *)_cffi_types[index])
 
 static PyObject *_cffi_init(const char *module_name, Py_ssize_t version,
                             const struct _cffi_type_context_s *ctx)
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.8.4"
+                               "\ncompiled with cffi version: 1.9.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -366,7 +366,7 @@
     const char *code;
 } _cffi_pypy_init = {
     _CFFI_MODULE_NAME,
-    _CFFI_PYTHON_STARTUP_FUNC,
+    (void(*)(const void *[]))_CFFI_PYTHON_STARTUP_FUNC,
     _CFFI_PYTHON_STARTUP_CODE,
 };
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -334,6 +334,8 @@
                         realtype, quals = self._get_type_and_quals(
                             decl.type, name=decl.name, partial_length_ok=True)
                     self._declare('typedef ' + decl.name, realtype, quals=quals)
+                elif decl.__class__.__name__ == 'Pragma':
+                    pass    # skip pragma, only in pycparser 2.15
                 else:
                     raise api.CDefError("unrecognized construct", decl)
         except api.FFIError as e:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,10 +519,18 @@
             smallest_value = min(self.enumvalues)
             largest_value = max(self.enumvalues)
         else:
-            raise api.CDefError("%r has no values explicitly defined: "
-                                "refusing to guess which integer type it is "
-                                "meant to be (unsigned/signed, int/long)"
-                % self._get_c_name())
+            import warnings
+            try:
+                # XXX!  The goal is to ensure that the warnings.warn()
+                # will not suppress the warning.  We want to get it
+                # several times if we reach this point several times.
+                __warningregistry__.clear()
+            except NameError:
+                pass
+            warnings.warn("%r has no values explicitly defined; "
+                          "guessing that it is equivalent to 'unsigned int'"
+                          % self._get_c_name())
+            smallest_value = largest_value = 0
         if smallest_value < 0:   # needs a signed type
             sign = 1
             candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -1,4 +1,5 @@
 import os
+import sys
 
 try:
     basestring
@@ -74,8 +75,13 @@
     Add py_limited_api to kwds if setuptools >= 26 is in use.
     Do not alter the setting if it already exists.
     Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+
+    CPython itself should ignore the flag in a debugging version
+    (by not listing .abi3.so in the extensions it supports), but
+    it doesn't so far, creating troubles.  That's why we check
+    for "not sys.flags.debug". (http://bugs.python.org/issue28401)
     """
-    if 'py_limited_api' not in kwds:
+    if 'py_limited_api' not in kwds and not sys.flags.debug:
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -9,8 +9,8 @@
   Philip Jenvey
   Ronan Lamy
   Brian Kearns
+  Richard Plangger
   Michael Hudson
-  Richard Plangger
   Manuel Jacob
   David Schneider
   Holger Krekel
@@ -54,10 +54,10 @@
   Jason Creighton
   Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
-  Spenser Bauman
   Tyler Wade
   Vincent Legoll
   Michael Foord
@@ -65,13 +65,13 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Stefano Rivera
   Timo Paulssen
   Edd Barrett
   Squeaky
@@ -89,12 +89,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -128,10 +128,11 @@
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -147,17 +148,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
   Daniel Patrick
-  Jasper Schulz
-  Christian Hudon
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -165,8 +165,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -175,11 +175,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -196,7 +197,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -219,6 +219,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -231,6 +232,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -258,9 +260,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -274,10 +274,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
-  "Aaron Gallagher
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -287,8 +287,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -302,13 +303,11 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
-  JohnDoe
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
   reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
@@ -316,25 +315,24 @@
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
   remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
   Roman Podoliaka
   Dan Loewenherz
   werat
-  hgattic
+
+
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -473,6 +473,10 @@
   time and the standard deviation, instead of the minimum, since the minimum is
   often misleading.
 
+* The ``get_config_vars`` method of ``sysconfig`` and ``distutils.sysconfig``
+  are not complete. On POSIX platforms, CPython fishes configuration variables
+  from the Makefile used to build the interpreter. PyPy should bake the values
+  in during compilation, but does not do that yet.
 
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -397,3 +397,28 @@
 in auto-generated C code, and at least some knowledge about the
 various components involved, from PyPy's own RPython source code to
 the GC and possibly the JIT.
+
+
+Why doesn't PyPy move to GitHub, Gitlab, ...?
+----------------------------------------------
+
+We've been quite happy with bitbucket.org. Moving version control systems and
+hosting is a lot of hard work: On the one hand, PyPy's mercurial history is
+long and gnarly. On the other hand, all our infrastructure (buildbots,
+benchmarking, etc) would have to be adapted. So unless somebody steps up and
+volunteers to do all that work, it will likely not happen.
+
+
+What is needed for Windows 64 support of PyPy?
+-----------------------------------------------
+
+First, please note that the Windows 32 PyPy binary works just fine on Windows
+64. The only problem is that it only supports up to 4GB of heap per process.
+
+As to real Windows 64 support: Currently we don't have an active PyPy developer
+whose main development platform is Windows. So if you are interested in getting
+Windows 64 support, we encourage you to volunteer `to make it happen`_! Another
+option would be to pay some PyPy developers to implement Windows 64 support,
+but so far there doesn't seem to be an overwhelming commercial interest in it.
+
+.. _`to make it happen`: windows.html#what-is-missing-for-a-full-64-bit-translation
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -1,4 +1,4 @@
-The PyPy Release Process
+PyPy's Release Process
 ========================
 
 Release Policy
@@ -39,41 +39,60 @@
   Some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
   necessary.
-* update pypy/doc/contributor.rst (and possibly LICENSE)
-  pypy/doc/tool/makecontributor.py generates the list of contributors
-* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  create a fresh whatsnew_head.rst after the release
-  and add the new file to  pypy/doc/index-of-whatsnew.rst
-* go to pypy/tool/release and run
-  ``force-builds.py <release branch>``
-  The following JIT binaries should be built, however, we need more buildbots
-  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
-  freebsd64 
 
-* wait for builds to complete, make sure there are no failures
-* download the builds, repackage binaries. Tag the release version
-  and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+* Update and write documentation
 
-  Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
-  and to cobra, as some packagers prefer a clearly labeled source package
-  ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
-  unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+  * update pypy/doc/contributor.rst (and possibly LICENSE)
+    pypy/doc/tool/makecontributor.py generates the list of contributors
 
-* Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+  * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+    create a fresh whatsnew_head.rst after the release
+    and add the new file to  pypy/doc/index-of-whatsnew.rst
 
-* write release announcement pypy/doc/release-x.y(.z).txt
+  * write release announcement pypy/doc/release-VERSION.rst
+    The release announcement should contain a direct link to the download page
 
-  The release announcement should contain a direct link to the download page
+  * Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
 
-* Add the new files to  pypy/doc/index-of-{whatsnew,release-notes}.rst
+* Build and upload the release tar-balls
 
-* update pypy.org (under extradoc/pypy.org), rebuild and commit
+  * go to pypy/tool/release and run
+    ``force-builds.py <release branch>``
+    The following JIT binaries should be built, however, we need more buildbots
+    windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+    freebsd64 
 
-* post announcement on morepypy.blogspot.com
-* send announcements to twitter.com, pypy-dev, python-list,
-  python-announce, python-dev ...
+  * wait for builds to complete, make sure there are no failures
 
-* add a tag on the pypy/jitviewer repo that corresponds to pypy release
-* add a tag on the codespeed web site that corresponds to pypy release
-* revise versioning at https://readthedocs.org/projects/pypy
+  * send out a mailing list message asking for people to test before uploading
+    to prevent having to upload more than once
+
+  * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
+    that the source tarball can be produced in the next steps
+
+  * download the builds, repackage binaries. Tag the release version
+    and download and repackage source from bitbucket. You may find it
+    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+
+    Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
+    and to cobra, as some packagers prefer a clearly labeled source package
+    ( download e.g.  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2,
+    unpack, rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
+  * Upload binaries to https://bitbucket.org/pypy/pypy/downloads
+
+* Send out a mailing list message asking for last-minute comments and testing
+
+* RELEASE !  
+
+  * update pypy.org (under extradoc/pypy.org), rebuild and commit, using the
+    hashes produced from the ``repackage.sh`` script or by hand
+
+  * post announcement on morepypy.blogspot.com
+  * send announcements to twitter.com, pypy-dev, python-list,
+    python-announce, python-dev ...
+
+* If all is OK, document the released version
+
+  * add a tag on the codespeed web site that corresponds to pypy release
+  * revise versioning at https://readthedocs.org/projects/pypy
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
    release-pypy2.7-v5.3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
    whatsnew-pypy2-5.3.1.rst
    whatsnew-pypy2-5.3.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -1,8 +1,17 @@
 Downloading and Installing PyPy
 ===============================
 
+Using a packaged PyPy
+~~~~~~~~~~~~~~~~~~~~~
+
+Some Linux distributions provide a pypy package. Note that in order to
+install additional modules that require compilation, you may need to install
+additional packages such as pypy-dev. This will manifest as an error about
+"missing Python.h". Distributions do not as of yet supply many pypy-ready
+packages, if you require additional modules we recommend creating a virtualenv
+and using pip. 
+
 .. _prebuilt-pypy:
-
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -38,6 +47,9 @@
 and not move the binary there, else PyPy would not be able to find its
 library.
 
+Installing more modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
 If you want to install 3rd party libraries, the most convenient way is
 to install pip_ using ensurepip_ (unless you want to install virtualenv as 
 explained below; then you can directly use pip inside virtualenvs):
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -6,17 +6,24 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.12.
 
 We continue to make incremental improvements to our C-API
-compatability layer (cpyext). We pass all but a few of the tests in the
+compatibility layer (cpyext). We pass all but a few of the tests in the
 upstream numpy `test suite`_. 
 
 Work proceeds at a good pace on the PyPy3.5
 version due to a grant_ from the Mozilla Foundation, and some of those
-changes have been backported to PyPy2.7 where relevant
+changes have been backported to PyPy2.7 where relevant.
+
+The PowerPC and s390x backend have been enhanced_ with the capability use SIMD instructions
+ for micronumpy loops.
 
 We changed ``timeit`` to now report average +- standard deviation, which is
 better than the misleading minimum value reported in CPython.
 
-XXX
+We now support building PyPy with OpenSSL 1.1 in our built-in _ssl module, as
+well as maintaining support for previous versions.
+
+CFFI_ has been updated to 1.9, improving an already great package for
+interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -35,12 +42,13 @@
 with making RPython's JIT even better.
 
 .. _`test suite`: https://bitbucket.org/pypy/pypy/wiki/Adventures%20in%20cpyext%20compatibility
-.. _cffi: https://cffi.readthedocs.org
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: http://doc.pypy.org
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+.. _`enhanced`: https://morepypy.blogspot.co.at/2016/11/vectorization-extended-powerpc-and-s390x.html
 
 What is PyPy?
 =============
@@ -64,42 +72,46 @@
   * **s390x** running Linux
 
 .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
-.. _`dynamic languages`: http://pypyjs.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
 
 Other Highlights (since 5.4 released Aug 31, 2016)
 =========================================================
 
 * New features
+
   * Allow tests run with `-A` to find `libm.so` even if it is a script not a
     dynamically loadable file
-  * Backport fixes to rposix on windows from py2.5
+  * Backport fixes to rposix on windows from py3.5
   * Allow user-defined ``__getitem__`` on subclasses of ``str`` and ``unicode``
   * Add ``inode`` to ``scandir()`` on posix systems
   * Support more attributes on ``super``
   * Issue #2386: non-latin1 unicode keys were ignored in ``unicode.format(**d)``
   * Restore the ability to translate with CPython
-  * Update to CFFI 1.8.4
+  * Update to CFFI 1.9.0
   * Support the new buffer protocol in cpyext and numpypy
   * Add ``rposix.sync()``
   * Support full-precision nanosecond times in os.stat()
   * Add documentation about the assembler backends to RPYthon
   * Search for the stdlibs from the libpypy shared object rather than the pypy-c exe,
-    changes downstream packaging requirments
+    changes downstream packaging requirements
   * Add ``try_inline``, like ``always_inline`` and ``dont_inline`` to RPython
   * Reject ``'a'.strip(buffer(' '))`` like cpython (the argument to strip must
     be ``str`` or ``unicode``)
   * Allow ``warning.warn(('something', 1), Warning)`` like on CPython
   * Refactor ``rclock`` and add some more ``CLOCK_xxx`` constants on
     relevant platforms
-  * Backport the ``'faulthandler`` module from py3.5
+  * Backport the ``faulthandler`` module from py3.5
   * Improve the error message when trying to call a method where the ``self``
     parameter is missing in the definition
   * Implement ``rposix.cpu_count``
   * Support translation on FreeBSD running on PowerPC
   * Implement ``__rmod__`` on ``str`` and ``unicode`` types
   * Issue warnings for stricter handling of ``__new__``, ``__init__`` args
+  * When using ``struct.unpack('q', ...`` try harder to prefer int to long
+  * Support OpenSSL version 1.1 (in addition to version 1.0)
 
 * Bug Fixes
+
   * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
   * Fix translation of the sandbox
   * Fix for an issue where `unicode.decode('utf8', 'custom_replace')` messed up
@@ -120,6 +132,7 @@
   * ``_numpypy.add.reduce`` returns a scalar now
 
 * Performance improvements:
+
   * Improve method calls on oldstyle classes
   * Clean and refactor code for testing cpyext to allow sharing with py3.5
   * Refactor a building the map of reflected ops in ``_numpypy``
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -110,4 +110,4 @@
         "hg log -r %s --template '{branch}'" % startrev)
     if errcode != 0:
         py.test.skip('no Mercurial repo')
-    assert wc_branch == 'default'
+    assert wc_branch in ('default', "'default'") # sometimes the ' leaks (windows)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -32,7 +32,7 @@
     'Niklaus Haldimann': ['nik'],
     'Alexander Schremmer': ['xoraxax'],
     'Anders Hammarquist': ['iko'],
-    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'David Edelsohn': ['edelsoh', 'edelsohn','opassembler.py'],
     'Niko Matsakis': ['niko'],
     'Jakub Gustak': ['jlg'],
     'Guido Wesdorp': ['guido'],
@@ -75,6 +75,9 @@
     'Spenser Bauman':['Spenser Andrew Bauman'],
     'Raffael Tfirst':['raffael.tfirst at gmail.com'],
     'timo':['timo at eistee.fritz.box'],
+    'Jasper Schulz':['Jasper.Schulz'],
+    'Aaron Gallagher':['"Aaron Gallagher'],
+    'Yasir Suhail':['yasirs'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,82 +1,16 @@
 ==========================
-What's new in PyPy2.7 5.4+
+What's new in PyPy2.7 5.6+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.4
-.. startrev: 522736f816dc
+.. this is a revision shortly after release-pypy2.7-v5.6
+.. startrev: 7e9787939641
 
-.. branch: rpython-resync
-Backport rpython changes made directly on the py3k and py3.5 branches.
+.. branch: rpython-error-to-systemerror
 
-.. branch: buffer-interface
-Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
-in numpypy
+Any uncaught RPython exception (from a PyPy bug) is turned into an
+app-level SystemError.  This should improve the lot of users hitting an
+uncaught RPython error.
 
-.. branch: force-virtual-state
-Improve merging of virtual states in the JIT in order to avoid jumping to the
-preamble. Accomplished by allocating virtual objects where non-virtuals are
-expected.
+.. branch: union-side-effects-2
 
-.. branch: conditional_call_value_3
-JIT residual calls: if the called function starts with a fast-path
-like "if x.foo != 0: return x.foo", then inline the check before
-doing the CALL.  For now, string hashing is about the only case.
-
-.. branch: search-path-from-libpypy
-
-The compiled pypy now looks for its lib-python/lib_pypy path starting
-from the location of the *libpypy-c* instead of the executable. This is
-arguably more consistent, and also it is what occurs anyway if you're
-embedding pypy.  Linux distribution packagers, take note!  At a minimum,
-the ``libpypy-c.so`` must really be inside the path containing
-``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
-from somewhere else.  You no longer have to do the same with the
-``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
-
-.. branch: _warnings
-
-CPython allows warning.warn(('something', 1), Warning), on PyPy this
-produced a "expected a readable buffer object" error. Test and fix.
-
-.. branch: stricter-strip
-
-CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
-allowed as arguments. Test and fix for str and unicode
-
-.. branch: faulthandler
-
-Port the 'faulthandler' module to PyPy default.  This module is standard
-in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
-
-.. branch: test-cpyext
-
-Refactor cpyext testing to be more pypy3-friendly.
-
-.. branch: better-error-missing-self
-
-Improve the error message when the user forgot the "self" argument of a method.
-
-
-.. fb6bb835369e
-Change the ``timeit`` module: it now prints the average time and the standard
-deviation over 7 runs by default, instead of the minimum. The minimum is often
-misleading.
-
-.. branch: unrecursive-opt
-
-Make optimiseopt iterative instead of recursive so it can be reasoned about
-more easily and debugging is faster.
-
-.. branch: Tiberiumk/fix-2412-1476011166874
-.. branch: redirect-assembler-jitlog
-
-
-
-.. branch: stdlib-2.7.12
-
-Update stdlib to version 2.7.12
-
-.. branch: buffer-interface2
-
-Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
-types still missing
+Try to improve the consistency of RPython annotation unions.
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -0,0 +1,107 @@
+=========================
+What's new in PyPy2.7 5.6
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.4
+.. startrev: 522736f816dc
+
+.. branch: rpython-resync
+Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
+
+.. branch: conditional_call_value_3
+JIT residual calls: if the called function starts with a fast-path
+like "if x.foo != 0: return x.foo", then inline the check before
+doing the CALL.  For now, string hashing is about the only case.
+
+.. branch: search-path-from-libpypy
+
+The compiled pypy now looks for its lib-python/lib_pypy path starting
+from the location of the *libpypy-c* instead of the executable. This is
+arguably more consistent, and also it is what occurs anyway if you're
+embedding pypy.  Linux distribution packagers, take note!  At a minimum,
+the ``libpypy-c.so`` must really be inside the path containing
+``lib-python`` and ``lib_pypy``.  Of course, you can put a symlink to it
+from somewhere else.  You no longer have to do the same with the
+``pypy`` executable, as long as it finds its ``libpypy-c.so`` library.
+
+.. branch: _warnings
+
+CPython allows warning.warn(('something', 1), Warning), on PyPy this
+produced a "expected a readable buffer object" error. Test and fix.
+
+.. branch: stricter-strip
+
+CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are
+allowed as arguments. Test and fix for str and unicode
+
+.. branch: faulthandler
+
+Port the 'faulthandler' module to PyPy default.  This module is standard
+in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI.
+
+.. branch: test-cpyext
+
+Refactor cpyext testing to be more pypy3-friendly.
+
+.. branch: better-error-missing-self
+
+Improve the error message when the user forgot the "self" argument of a method.
+
+
+.. fb6bb835369e
+Change the ``timeit`` module: it now prints the average time and the standard
+deviation over 7 runs by default, instead of the minimum. The minimum is often
+misleading.
+
+.. branch: unrecursive-opt
+
+Make optimiseopt iterative instead of recursive so it can be reasoned about
+more easily and debugging is faster.
+
+.. branch: Tiberiumk/fix-2412-1476011166874
+.. branch: redirect-assembler-jitlog
+
+
+
+.. branch: stdlib-2.7.12
+
+Update stdlib to version 2.7.12
+
+.. branch: buffer-interface2
+
+Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
+types still missing
+
+
+.. branch: fix-struct-unpack-Q
+
+Improve compatibility with CPython in the ``struct`` module. In particular,
+``struct.unpack`` now returns an ``int`` whenever the returned value fits,
+while previously it always returned a ``long`` for certains format codes such
+as ``Q`` (and also ``I``, ``L`` and ``q`` on 32 bit)
+
+.. branch: zarch-simd-support
+
+s390x implementation for vector operations used in VecOpt
+
+.. branch: ppc-vsx-support
+
+PowerPC implementation for vector operations used in VecOpt
+
+.. branch: newinitwarn
+
+Match CPython's stricter handling of __new/init__ arguments
+
+.. branch: openssl-1.1
+
+Support for OpenSSL version 1.1 (in addition to version 1.0).
+Tested on Linux (1.1, 1.0), on Win32, and Mac (1.0 only)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -20,6 +20,22 @@
 
 .. _our downloads: http://pypy.org/download.html
 
+Installing Visual Compiler v9 (for Python 2.7)
+----------------------------------------------
+
+This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
+was checked in Nov 2016). Note that the compiler suite will be installed in
+``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
+Using a current version of ``setuptools`` will be able to find it there. For
+Windows 10, you must right-click the download, and under ``Properties`` ->
+``Compatibility`` mark it as ``Run run this program in comatibility mode for``
+``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
+otherwise ``mt.exe`` will silently fail. Installation will begin automatically
+by running the mt.exe command by hand from a DOS window (that is how the author
+discovered the problem).
+
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
 
 Translating PyPy with Visual Studio
 -----------------------------------
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -33,7 +33,12 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
+        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
+    else:
+        w_initstdio = space.appexec([], """():
+            return lambda unbuffered: None
+        """)
 
     def entry_point(argv):
         if withjit:
@@ -103,18 +108,23 @@
                       " not found in %s or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
-        space.appexec([w_path], """(path):
-            import sys
-            sys.path[:] = path
-        """)
-        # import site
+        must_leave = space.threadlocals.try_enter_thread(space)
         try:
-            space.setattr(space.getbuiltinmodule('sys'),
-                          space.wrap('executable'),
-                          space.wrap(home))
-            import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
-                                    space.wrap('__import__'))
-            space.call_function(import_, space.wrap('site'))
+            # initialize sys.{path,executable,stdin,stdout,stderr}
+            # (in unbuffered mode, to avoid troubles) and import site
+            space.appexec([w_path, space.wrap(home), w_initstdio],
+            r"""(path, home, initstdio):
+                import sys
+                sys.path[:] = path
+                sys.executable = home
+                initstdio(unbuffered=True)
+                try:
+                    import site
+                except Exception as e:
+                    sys.stderr.write("'import site' failed:\n")
+                    import traceback
+                    traceback.print_exc()
+            """)
             return rffi.cast(rffi.INT, 0)
         except OperationError as e:
             if verbose:
@@ -122,6 +132,9 @@
                 debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
+        finally:
+            if must_leave:
+                space.threadlocals.leave_thread(space)
 
     @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
@@ -240,7 +253,8 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if config.translation.profopt is not None:
+            if (config.translation.profopt is not None
+                    and not config.translation.noprofopt):
                 raise Exception("Cannot use the --profopt option "
                                 "when --shared is on (it is by default). "
                                 "See issue #2398.")
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -275,6 +275,10 @@
 def set_fully_buffered_io():
     sys.stdout = sys.__stdout__ = fdopen(1, 'w')
 
+def initstdio(unbuffered=False):
+    if unbuffered:
+        set_unbuffered_io()
+
 # ____________________________________________________________
 # Main entry point
 
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,6 +7,7 @@
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib import rstackovf
 
 from pypy.interpreter import debug
 
@@ -58,10 +59,14 @@
     def __str__(self):
         "NOT_RPYTHON: Convenience for tracebacks."
         s = self._w_value
-        if self.__class__ is not OperationError and s is None:
-            space = getattr(self.w_type, 'space')
-            if space is not None:
+        space = getattr(self.w_type, 'space', None)
+        if space is not None:
+            if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
+            try:
+                s = space.str_w(s)
+            except Exception:
+                pass
         return '[%s: %s]' % (self.w_type, s)
 
     def errorstr(self, space, use_repr=False):
@@ -73,11 +78,8 @@
             exc_value = str(w_value)
         else:
             w = space.wrap
-            if space.is_w(space.type(self.w_type), space.w_str):
-                exc_typename = space.str_w(self.w_type)
-            else:
-                exc_typename = space.str_w(
-                    space.getattr(self.w_type, w('__name__')))
+            exc_typename = space.str_w(
+                space.getattr(self.w_type, w('__name__')))
             if space.is_w(w_value, space.w_None):
                 exc_value = ""
             else:
@@ -508,3 +510,47 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+ at jit.dont_look_inside
+def get_converted_unexpected_exception(space, e):
+    """This is used in two places when we get an non-OperationError
+    RPython exception: from gateway.py when calling an interp-level
+    function raises; and from pyopcode.py when we're exiting the
+    interpretation of the frame with an exception.  Note that it
+    *cannot* be used in pyopcode.py: that place gets a
+    ContinueRunningNormally exception from the JIT, which must not end
+    up here!
+    """
+    try:
+        if not we_are_translated():
+            raise
+        raise e
+    except KeyboardInterrupt:
+        return OperationError(space.w_KeyboardInterrupt, space.w_None)
+    except MemoryError:
+        return OperationError(space.w_MemoryError, space.w_None)
+    except rstackovf.StackOverflow as e:
+        # xxx twisted logic which happens to give the result that we
+        # want: when untranslated, a RuntimeError or its subclass
+        # NotImplementedError is caught here.  Then
+        # check_stack_overflow() will re-raise it directly.  We see
+        # the result as this exception propagates directly.  But when
+        # translated, an RPython-level RuntimeError is turned into
+        # an app-level RuntimeError by the next case.
+        rstackovf.check_stack_overflow()
+        return oefmt(space.w_RuntimeError,
+                     "maximum recursion depth exceeded")
+    except RuntimeError:   # not on top of py.py
+        return OperationError(space.w_RuntimeError, space.w_None)
+    except:
+        if we_are_translated():
+            from rpython.rlib.debug import debug_print_traceback
+            debug_print_traceback()
+            extra = '; internal traceback was dumped to stderr'
+        else:
+            # when untranslated, we don't wrap into an app-level
+            # SystemError (this makes debugging tests harder)
+            raise
+        return OperationError(space.w_SystemError, space.wrap(
+            "unexpected internal exception (please report a bug): %r%s" %
+            (e, extra)))
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -611,7 +611,7 @@
 
 class StaticMethod(W_Root):
     """The staticmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -622,13 +622,16 @@
 
     def descr_staticmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(StaticMethod, w_subtype)
-        instance.__init__(w_function)
-        return space.wrap(instance)
+        instance.__init__(space.w_None)
+        return instance
+
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
 
 
 class ClassMethod(W_Root):
     """The classmethod objects."""
-    _immutable_fields_ = ['w_function']
+    _immutable_fields_ = ['w_function?']
 
     def __init__(self, w_function):
         self.w_function = w_function
@@ -641,8 +644,11 @@
 
     def descr_classmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(ClassMethod, w_subtype)
-        instance.__init__(w_function)
-        return space.wrap(instance)
+        instance.__init__(space.w_None)
+        return instance
+
+    def descr_init(self, space, w_function):
+        self.w_function = w_function
 
 class FunctionWithFixedCode(Function):
     can_change_code = False
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -23,7 +23,6 @@
     DescrMismatch)
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode
-from rpython.rlib import rstackovf
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint
 from rpython.tool.sourcetools import func_with_new_name, compile2
@@ -712,16 +711,11 @@
             if not we_are_translated():
                 raise
             raise e
-        except KeyboardInterrupt:
-            raise OperationError(space.w_KeyboardInterrupt, space.w_None)
-        except MemoryError:
-            raise OperationError(space.w_MemoryError, space.w_None)
-        except rstackovf.StackOverflow as e:
-            rstackovf.check_stack_overflow()
-            raise oefmt(space.w_RuntimeError,
-                        "maximum recursion depth exceeded")
-        except RuntimeError:   # not on top of py.py
-            raise OperationError(space.w_RuntimeError, space.w_None)
+        except OperationError:
+            raise
+        except Exception as e:      # general fall-back
+            from pypy.interpreter import error
+            raise error.get_converted_unexpected_exception(space, e)
 
 # (verbose) performance hack below
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -1,6 +1,7 @@
 """ PyFrame class implementation with the interpreter main loop.
 """
 
+import sys
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint
@@ -278,6 +279,10 @@
                         self.pushvalue(w_inputvalue)
                 w_exitvalue = self.dispatch(self.pycode, next_instr,
                                             executioncontext)
+            except OperationError:
+                raise
+            except Exception as e:      # general fall-back
+                raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None
@@ -883,6 +888,14 @@
             frame = frame.f_backref()
         return None
 
+    def _convert_unexpected_exception(self, e):
+        from pypy.interpreter import error
+
+        operr = error.get_converted_unexpected_exception(self.space, e)
+        pytraceback.record_application_traceback(
+            self.space, operr, self, self.last_instr)
+        raise operr
+
 # ____________________________________________________________
 
 def get_block_class(opname):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -1,8 +1,10 @@
 import py
 from pypy.interpreter import executioncontext
+from pypy.interpreter.error import OperationError
 
-class Finished(Exception):
-    pass
+class Finished(OperationError):
+    def __init__(self):
+        OperationError.__init__(self, "exception_class", "exception_value")
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -4,6 +4,7 @@
 from pypy.interpreter import gateway, argument
 from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.error import OperationError
 import py
 import sys
 
@@ -771,6 +772,42 @@
         w_g = space.wrap(gateway.interp2app_temp(g, doc='bar'))
         assert space.unwrap(space.getattr(w_g, space.wrap('__doc__'))) == 'bar'
 
+    def test_system_error(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        e = py.test.raises(OperationError, space.appexec, [w_g], """(my_g):
+            my_g()
+        """)
+        err = str(e.value)
+        assert 'SystemError' in err
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
+    def test_system_error_2(self):
+        py.test.skip("we don't wrap a random exception inside SystemError "
+                     "when untranslated, because it makes testing harder")
+        class UnexpectedException(Exception):
+            pass
+        space = self.space
+        def g(space):
+            raise UnexpectedException
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_msg = space.appexec([w_g], """(my_g):
+            try:
+                my_g()
+            except SystemError as e:
+                return str(e)
+        """)
+        err = space.str_w(w_msg)
+        assert ('unexpected internal exception (please '
+                'report a bug): UnexpectedException') in err
+
 
 class AppTestPyTestMark:
     @py.test.mark.unlikely_to_exist
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -20,7 +20,7 @@
                                                 space.wrap('modules')),
                                                 space.wrap('xyz')))
     assert x == 3
-    lls = rffi.str2charp("sys")
+    lls = rffi.str2charp("sys   # should give a NameError")
     execute_source(lls)
     lltype.free(lls, flavor='raw')
     # did not crash - the same globals
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -106,8 +106,10 @@
 # So we create a few interp-level subclasses of W_XxxObject, which add
 # some combination of features. This is done using mapdict.
 
-# we need two subclasses of the app-level type, one to add mapdict, and then one
-# to add del to not slow down the GC.
+# Note that nowadays, we need not "a few" but only one subclass.  It
+# adds mapdict, which flexibly allows all features.  We handle the
+# presence or absence of an app-level '__del__' by calling
+# register_finalizer() or not.
 
 @specialize.memo()
 def get_unique_interplevel_subclass(space, cls):
@@ -686,15 +688,17 @@
 (e.g. C().f()).  The instance is ignored except for its class.""",
     __get__ = interp2app(StaticMethod.descr_staticmethod_get),
     __new__ = interp2app(StaticMethod.descr_staticmethod__new__.im_func),
+    __init__=interp2app(StaticMethod.descr_init),
     __func__= interp_attrproperty_w('w_function', cls=StaticMethod),
     )
 
 ClassMethod.typedef = TypeDef(
     'classmethod',
-    __new__ = interp2app(ClassMethod.descr_classmethod__new__.im_func),
-    __get__ = interp2app(ClassMethod.descr_classmethod_get),
-    __func__= interp_attrproperty_w('w_function', cls=ClassMethod),
-    __doc__ = """classmethod(function) -> class method
+    __new__=interp2app(ClassMethod.descr_classmethod__new__.im_func),
+    __init__=interp2app(ClassMethod.descr_init),
+    __get__=interp2app(ClassMethod.descr_classmethod_get),
+    __func__=interp_attrproperty_w('w_function', cls=ClassMethod),
+    __doc__="""classmethod(function) -> class method
 
 Convert a function to be a class method.
 
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -509,59 +509,6 @@
         assert eval("i", None, None) == 4
         assert eval('a', None, dict(a=42)) == 42
 
-    def test_compile(self):
-        co = compile('1+2', '?', 'eval')
-        assert eval(co) == 3
-        co = compile(buffer('1+2'), '?', 'eval')
-        assert eval(co) == 3
-        exc = raises(TypeError, compile, chr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
-        assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
-        compile("from __future__ import with_statement", "<test>", "exec")
-        raises(SyntaxError, compile, '-', '?', 'eval')
-        raises(ValueError, compile, '"\\xt"', '?', 'eval')
-        raises(ValueError, compile, '1+2', '?', 'maybenot')
-        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
-        raises(TypeError, compile, '1+2', 12, 34)
-
-    def test_compile_error_message(self):
-        import re
-        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
-        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
-        exc = raises(SyntaxError, compile,
-            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
-        assert 'iso-8859-15' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-        exc = raises(SyntaxError, compile,
-            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
-        assert 'fake' in str(exc.value)
-        assert 'BOM' in str(exc.value)
-
-    def test_unicode_compile(self):
-        try:
-            compile(u'-', '?', 'eval')
-        except SyntaxError as e:
-            assert e.lineno == 1
-
-    def test_unicode_encoding_compile(self):
-        code = u"# -*- coding: utf-8 -*-\npass\n"
-        raises(SyntaxError, compile, code, "tmp", "exec")
-
-    def test_recompile_ast(self):
-        import _ast
-        # raise exception when node type doesn't match with compile mode
-        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
-        raises(TypeError, compile, co1, '<ast>', 'eval')
-        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
-
     def test_isinstance(self):
         assert isinstance(5, int)
         assert isinstance(5, object)
@@ -624,34 +571,10 @@
         raises(TypeError, hasattr, x, 42)
         raises(UnicodeError, hasattr, x, u'\u5678')  # cannot encode attr name
 
-    def test_compile_leading_newlines(self):
-        src = """
-def fn(): pass
-"""
-        co = compile(src, 'mymod', 'exec')
-        firstlineno = co.co_firstlineno
-        assert firstlineno == 2
-
-    def test_compile_null_bytes(self):
-        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        src = "#abc\x00def\n"
-        raises(TypeError, compile, src, 'mymod', 'exec')
-        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+    def test_execfile_args(self):
         execfile(self.nullbytes) # works
-
-    def test_execfile_args(self):
         raises(TypeError, execfile, self.nonexistent, {}, ())
 
-    def test_compile_null_bytes_flag(self):
-        try:
-            from _ast import PyCF_ACCEPT_NULL_BYTES
-        except ImportError:
-            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               PyCF_ACCEPT_NULL_BYTES)
-        src = "#abc\x00def\n"
-        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
-
     def test_print_function(self):
         import __builtin__
         import sys
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -0,0 +1,77 @@
+class AppTestCompile:
+    def test_simple(self):
+        co = compile('1+2', '?', 'eval')
+        assert eval(co) == 3
+        co = compile(buffer('1+2'), '?', 'eval')
+        assert eval(co) == 3
+        exc = raises(TypeError, compile, chr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, unichr(0), '?', 'eval')
+        assert str(exc.value) == "compile() expected string without null bytes"
+        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
+        assert str(exc.value) == "expected a readable buffer object"
+        compile("from __future__ import with_statement", "<test>", "exec")
+        raises(SyntaxError, compile, '-', '?', 'eval')
+        raises(ValueError, compile, '"\\xt"', '?', 'eval')
+        raises(ValueError, compile, '1+2', '?', 'maybenot')
+        raises(ValueError, compile, "\n", "<string>", "exec", 0xff)
+        raises(TypeError, compile, '1+2', 12, 34)
+
+    def test_error_message(self):
+        import re
+        compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
+        compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
+        exc = raises(SyntaxError, compile,
+            b'# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
+        assert 'iso-8859-15' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+        exc = raises(SyntaxError, compile,
+            b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
+        assert 'fake' in str(exc.value)
+        assert 'BOM' in str(exc.value)
+
+    def test_unicode(self):
+        try:
+            compile(u'-', '?', 'eval')
+        except SyntaxError as e:
+            assert e.lineno == 1
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        raises(SyntaxError, compile, code, "tmp", "exec")
+
+    def test_recompile_ast(self):
+        import _ast
+        # raise exception when node type doesn't match with compile mode
+        co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
+        raises(TypeError, compile, co1, '<ast>', 'eval')
+        co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
+        compile(co2, '<ast>', 'eval')
+
+    def test_leading_newlines(self):
+        src = """
+def fn(): pass
+"""
+        co = compile(src, 'mymod', 'exec')
+        firstlineno = co.co_firstlineno
+        assert firstlineno == 2
+
+    def test_null_bytes(self):
+        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        src = "#abc\x00def\n"
+        raises(TypeError, compile, src, 'mymod', 'exec')
+        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+
+    def test_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -20,6 +20,12 @@
         x = Static(1)
         assert isinstance(x, Static)
 
+        class C(Static):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_classmethod(self):
         class C(object):
             def f(cls, stuff):
@@ -41,8 +47,14 @@
         x = Classm(1)
         assert isinstance(x, Classm)
 
+        class C(Classm):
+            def __init__(self, callable):
+                super(C, self).__init__(callable)
+        y = C(1)
+        assert isinstance(y, C)
+
     def test_property_simple(self):
-        
+
         class a(object):
             def _get(self): return 42
             def _set(self, value): raise AttributeError
@@ -98,7 +110,7 @@
             assert message.startswith('super(type, obj): obj must be an instance or subtype of type')
 

From pypy.commits at gmail.com  Wed Nov 16 03:50:06 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 00:50:06 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <582c1dbe.0b561c0a.efd49.9dce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88406:b35f9dc08af7
Date: 2016-11-16 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b35f9dc08af7/

Log:	merge default into branch

diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -90,6 +90,10 @@
 Do CPython Extension modules work with PyPy?
 --------------------------------------------
 
+**First note that some Linux distributions (e.g. Ubuntu, Debian) split
+PyPy into several packages.  If you installed a package called "pypy",
+then you may also need to install "pypy-dev" for the following to work.**
+
 We have experimental support for CPython extension modules, so
 they run with minor changes.  This has been a part of PyPy since
 the 1.4 release, but support is still in beta phase.  CPython
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -11,6 +11,13 @@
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
+def PyAnySet_Check(space, w_obj):
+    """Return true if obj is a set object, a frozenset object, or an
+    instance of a subtype."""
+    return (space.isinstance_w(w_obj, space.gettypefor(W_SetObject)) or
+            space.isinstance_w(w_obj, space.gettypefor(W_FrozensetObject)))
+
+ at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyAnySet_CheckExact(space, w_obj):
     """Return true if obj is a set object or a frozenset object but
     not an instance of a subtype."""
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1476,18 +1476,6 @@
     raise NotImplementedError
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyAnySet_Check(space, p):
-    """Return true if p is a set object, a frozenset object, or an
-    instance of a subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyAnySet_CheckExact(space, p):
-    """Return true if p is a set object or a frozenset object but
-    not an instance of a subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyFrozenSet_CheckExact(space, p):
     """Return true if p is a frozenset object but not an instance of a
     subtype."""
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -41,11 +41,19 @@
         api.PySet_Clear(w_set)
         assert space.len_w(w_set) == 0
 
-    def test_anyset_checkexact(self, space, api):
+    def test_anyset_check(self, space, api):
         w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         w_frozenset = space.newfrozenset([space.wrap(i) for i in [1, 2, 3, 4]])
         assert api.PyAnySet_CheckExact(w_set)
         assert api.PyAnySet_CheckExact(w_frozenset)
+        assert api.PyAnySet_Check(w_set)
+        assert api.PyAnySet_Check(w_frozenset)
+        w_instance = space.appexec([], """():
+            class MySet(set):
+                pass
+            return MySet()
+        """)
+        assert api.PyAnySet_Check(w_instance)
 
 class AppTestSetObject(AppTestCpythonExtensionBase):
     def test_set_macro_cast(self):

From pypy.commits at gmail.com  Wed Nov 16 05:07:10 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 16 Nov 2016 02:07:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: updated hgignore to not displya cffi
 built stdlib modules
Message-ID: <582c2fce.c6bdc20a.e3d52.0ed3@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88407:afa55ff3c551
Date: 2016-11-16 11:06 +0100
http://bitbucket.org/pypy/pypy/changeset/afa55ff3c551/

Log:	updated hgignore to not displya cffi built stdlib modules added new
	files in lib_pypy/openssl/*

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -78,3 +78,9 @@
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
+^lib_pypy/_.*cffi.*.c$
+^lib_pypy/_.*cffi.*.o$
+^lib_pypy/_.*cffi.*.so$
+^lib_pypy/_openssl.c$
+^lib_pypy/_openssl.o$
+^lib_pypy/_openssl.*.so$
diff --git a/lib-python/3/test/wrongcert.pem b/lib-python/3/test/wrongcert.pem
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/wrongcert.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH
+FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T
+f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB
+AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq
+1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW
+7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg
+SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe
+19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg
+ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666
+lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs
+ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv
+frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk
+2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6
++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK
+24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G
+A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9
+OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt
+U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ
+fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E
+usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+
+43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw
+eSHj5jpC8iZKjCHBn+mAi4cQ514=
+-----END CERTIFICATE-----
diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -26,6 +26,7 @@
 static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE;
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS;
 static const long Cryptography_HAS_NPN_NEGOTIATED;
+static const long Cryptography_NO_TLSEXT;
 
 static const long Cryptography_OPENSSL_NPN_NEGOTIATED;
 
@@ -693,6 +694,12 @@
 
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1;
 
+#ifdef OPENSSL_NO_TLSEXT
+static const long Cryptography_NO_TLSEXT = 0;
+#else
+static const long Cryptography_NO_TLSEXT = 1;
+#endif
+
 #ifdef OPENSSL_NPN_NEGOTIATED
 static const long Cryptography_OPENSSL_NPN_NEGOTIATED = OPENSSL_NPN_NEGOTIATED;
 static const long Cryptography_HAS_NPN_NEGOTIATED = 1;
diff --git a/lib_pypy/openssl/_cffi_src/openssl/tls1.py b/lib_pypy/openssl/_cffi_src/openssl/tls1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/openssl/_cffi_src/openssl/tls1.py
@@ -0,0 +1,29 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/tls1.h>
+"""
+
+TYPES = """
+"""
+
+FUNCTIONS = """
+long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX *, int(*)(SSL*,int*,void*));
+long Cryptography_SSL_CTX_set_tlsext_servername_arg(SSL_CTX *, void * arg);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX * ctx, int(*cb)(SSL*,int*,void*)) {
+    return SSL_CTX_set_tlsext_servername_callback(ctx, cb);
+}
+long Cryptography_SSL_CTX_set_tlsext_servername_arg(SSL_CTX * ctx, void *arg) {
+    return SSL_CTX_set_tlsext_servername_arg(ctx, arg);
+}
+"""
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -205,7 +205,8 @@
         lib.ERR_clear_error()
         self.ssl = ssl = lib.SSL_new(ctx)
 
-        lib.SSL_set_app_data(ssl, b"")
+        self._app_data_handle = ffi.new_handle(self) # TODO release later
+        lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
         if sock:
             lib.SSL_set_fd(ssl, sock.fileno())
         else:
@@ -254,13 +255,25 @@
         self.ssl = ffi.NULL
         self.shutdown_seen_zero = 0
         self.handshake_done = 0
-        self.owner = None
+        self._owner = None
         self.server_hostname = None
         self.socket = None
         self.alpn_protocols = ffi.NULL
         self.npn_protocols = ffi.NULL
 
     @property
+    def owner(self):
+        if self._owner is None:
+            return None
+        return self._owner()
+
+    @owner.setter
+    def owner(self, value):
+        if value is None:
+            self._owner = None
+        self._owner = weakref.ref(value)
+
+    @property
     def context(self):
         return self.ctx
 
@@ -696,7 +709,8 @@
 
 class _SSLContext(object):
     __slots__ = ('ctx', '_check_hostname', 'servername_callback',
-                 'alpn_protocols', 'npn_protocols')
+                 'alpn_protocols', 'npn_protocols', 'set_hostname',
+                 '_set_hostname_handle')
 
     def __new__(cls, protocol):
         self = object.__new__(cls)
@@ -858,8 +872,9 @@
                 else:
                     raise TypeError("password should be a string or callable")
 
+            handle = ffi.new_handle(pw_info) # XXX MUST NOT be garbage collected
             lib.SSL_CTX_set_default_passwd_cb(self.ctx, Cryptography_pem_password_cb)
-            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, ffi.new_handle(pw_info))
+            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, handle)
 
         try:
             ffi.errno = 0
@@ -1099,20 +1114,21 @@
             lib.EC_KEY_free(key)
 
     def set_servername_callback(self, callback):
+        if not HAS_SNI or lib.Cryptography_NO_TLSEXT:
+            raise NotImplementedError("The TLS extension servername callback, "
+                    "SSL_CTX_set_tlsext_servername_callback, "
+                    "is not in the current OpenSSL library.")
         if callback is None:
             lib.SSL_CTX_set_tlsext_servername_callback(self.ctx, ffi.NULL)
-            self.servername_callback = None
+            self.set_hostname = None
+            self._set_hostname_handle = None
             return
         if not callable(callback):
             raise TypeError("not a callable object")
-        callback_struct = ServernameCallback()
-        callback_struct.ctx = self
-        callback_struct.set_hostname = callback
-        self.servername_callback = callback_struct
-        index = id(self)
-        SERVERNAME_CALLBACKS[index] = callback_struct
+        self.set_hostname = callback
+        self._set_hostname_handle = ffi.new_handle(self)
         lib.Cryptography_SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
-        lib.Cryptography_SSL_CTX_set_tlsext_servername_arg(self.ctx, ffi.new_handle(callback_struct))
+        lib.Cryptography_SSL_CTX_set_tlsext_servername_arg(self.ctx, self._set_hostname_handle)
 
     def _set_alpn_protocols(self, protos):
         if HAS_ALPN:
@@ -1137,75 +1153,83 @@
 
 
 
- at ffi.callback("void(void)")
-def _servername_callback(ssl, ad, arg):
-    struct = ffi.from_handle(arg)
-    w_ctx = struct.w_ctx
-    space = struct.space
-    w_callback = struct.w_set_hostname
-    if not w_ctx.servername_callback:
-        # Possible race condition.
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    # The high-level ssl.SSLSocket object
-    index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl))
-    w_ssl = SOCKET_STORAGE.get(index)
-    assert isinstance(w_ssl, SSLSocket)
-    # The servername callback expects an argument that represents the current
-    # SSL connection and that has a .context attribute that can be changed to
-    # identify the requested hostname. Since the official API is the Python
-    # level API we want to pass the callback a Python level object rather than
-    # a _ssl.SSLSocket instance. If there's an "owner" (typically an
-    # SSLObject) that will be passed. Otherwise if there's a socket then that
-    # will be passed. If both do not exist only then the C-level object is
-    # passed.
-    if w_ssl.w_owner is not None:
-        w_ssl_socket = w_ssl.w_owner()
-    elif w_ssl.w_socket is not None:
-        w_ssl_socket = w_ssl.w_socket()
-    else:
-        w_ssl_socket = w_ssl
-    if space.is_none(w_ssl_socket):
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
+if HAS_SNI and not lib.Cryptography_NO_TLSEXT:
+    @ffi.callback("int(SSL*,int*,void*)")
+    def _servername_callback(s, al, arg):
+        ssl_ctx = ffi.from_handle(arg)
+        servername = lib.SSL_get_servername(s, lib.TLSEXT_NAMETYPE_host_name)
+    #ifdef WITH_THREAD
+        # TODO PyGILState_STATE gstate = PyGILState_Ensure();
+    #endif
 
-    servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name)
-    try:
-        if not servername:
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket, space.w_None, w_ctx)
+        if ssl_ctx.set_hostname is None:
+            #/* remove race condition in this the call back while if removing the
+            # * callback is in progress */
+    #ifdef WITH_THREAD
+            # TODO PyGILState_Release(gstate);
+    #endif
+            return lib.SSL_TLSEXT_ERR_OK
+        #}
 
+        ssl = ffi.from_handle(lib.SSL_get_app_data(s))
+        assert isinstance(ssl, _SSLSocket)
+
+        # The servername callback expects an argument that represents the current
+        # SSL connection and that has a .context attribute that can be changed to
+        # identify the requested hostname. Since the official API is the Python
+        # level API we want to pass the callback a Python level object rather than
+        # a _ssl.SSLSocket instance. If there's an "owner" (typically an
+        # SSLObject) that will be passed. Otherwise if there's a socket then that
+        # will be passed. If both do not exist only then the C-level object is
+        # passed.
+        ssl_socket = ssl.owner
+        if not ssl_socket:
+            ssl_socket = ssl.get_socket_or_None()
+
+        if ssl_socket is None:
+            al[0] = lib.SSL_AD_INTERNAL_ERROR
+            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+
+        if servername == ffi.NULL:
+            try:
+                result = ssl_ctx.set_hostname(ssl_socket, None, ssl_ctx)
+            except:
+                # TODO
+                #        PyErr_WriteUnraisable(ssl_ctx->set_hostname);
+                al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
+                return lib.SSL_TLSEXT_ERR_ALERT_FATAL
         else:
-            w_servername = space.newbytes(rffi.charp2str(servername))
+            servername = ffi.string(servername)
+
             try:
-                w_servername_idna = space.call_method(
-                    w_servername, 'decode', space.wrap('idna'))
-            except OperationError as e:
-                e.write_unraisable(space, "undecodable server name")
-                ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-                return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
+                servername_idna = servername.decode("idna")
+            except UnicodeDecodeError:
+                raise # TODO?
+    #            PyErr_WriteUnraisable(servername_o);
 
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket,
-                                           w_servername_idna, w_ctx)
-    except OperationError as e:
-        e.write_unraisable(space, "in servername callback")
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_HANDSHAKE_FAILURE)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
+            try:
+                result = ssl_ctx.set_hostname(ssl_socket, servername_idna, ssl_ctx)
+            except:
+                # TODO
+                #        PyErr_WriteUnraisable(ssl_ctx->set_hostname);
+                al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
+                return lib.SSL_TLSEXT_ERR_ALERT_FATAL
 
-    if space.is_none(w_result):
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    else:
-        try:
-            ad[0] = rffi.cast(rffi.INT, space.int_w(w_result))
-        except OperationError as e:
-            e.write_unraisable(space, "servername callback result")
-            ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
+        if result is not None:
+            try:
+                al[0] = int(result)
+            except:
+    #                PyErr_WriteUnraisable(result);
+               al[0] = lib.SSL_AD_INTERNAL_ERROR
+               return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+        else:
+            # TODO gil state release?
+            return lib.SSL_TLSEXT_ERR_OK
 
 class ServernameCallback(object):
     ctx = None
 SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
+TEST = None
 
 def _asn1obj2py(obj):
     nid = lib.OBJ_obj2nid(obj)
@@ -1216,7 +1240,7 @@
     buf = ffi.new("char[255]")
     length = lib.OBJ_obj2txt(buf, len(buf), obj, 1)
     if length < 0:
-        _setSSLError("todo")
+        ssl_error(None)
     if length > 0:
         return (nid, sn, ln, _str_with_len(buf, length))
     else:
diff --git a/lib_pypy/openssl/_stdssl/errorcodes.py b/lib_pypy/openssl/_stdssl/errorcodes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/openssl/_stdssl/errorcodes.py
@@ -0,0 +1,394 @@
+# File generated by tools/make_ssl_data.py
+# Generated on 2016-11-10T17:38:59.402032
+
+from _openssl import ffi, lib 
+_lib_codes = []
+_lib_codes.append(("PEM", lib.ERR_LIB_PEM))
+_lib_codes.append(("SSL", lib.ERR_LIB_SSL))
+_lib_codes.append(("X509", lib.ERR_LIB_X509))
+_error_codes = []
+_error_codes.append(("BAD_BASE64_DECODE", lib.ERR_LIB_PEM, 100))
+_error_codes.append(("BAD_DECRYPT", lib.ERR_LIB_PEM, 101))
+_error_codes.append(("BAD_END_LINE", lib.ERR_LIB_PEM, 102))
+_error_codes.append(("BAD_IV_CHARS", lib.ERR_LIB_PEM, 103))
+_error_codes.append(("BAD_MAGIC_NUMBER", lib.ERR_LIB_PEM, 116))
+_error_codes.append(("BAD_PASSWORD_READ", lib.ERR_LIB_PEM, 104))
+_error_codes.append(("BAD_VERSION_NUMBER", lib.ERR_LIB_PEM, 117))
+_error_codes.append(("BIO_WRITE_FAILURE", lib.ERR_LIB_PEM, 118))
+_error_codes.append(("CIPHER_IS_NULL", lib.ERR_LIB_PEM, 127))
+_error_codes.append(("ERROR_CONVERTING_PRIVATE_KEY", lib.ERR_LIB_PEM, 115))
+_error_codes.append(("EXPECTING_PRIVATE_KEY_BLOB", lib.ERR_LIB_PEM, 119))
+_error_codes.append(("EXPECTING_PUBLIC_KEY_BLOB", lib.ERR_LIB_PEM, 120))
+_error_codes.append(("INCONSISTENT_HEADER", lib.ERR_LIB_PEM, 121))
+_error_codes.append(("KEYBLOB_HEADER_PARSE_ERROR", lib.ERR_LIB_PEM, 122))
+_error_codes.append(("KEYBLOB_TOO_SHORT", lib.ERR_LIB_PEM, 123))
+_error_codes.append(("NOT_DEK_INFO", lib.ERR_LIB_PEM, 105))
+_error_codes.append(("NOT_ENCRYPTED", lib.ERR_LIB_PEM, 106))
+_error_codes.append(("NOT_PROC_TYPE", lib.ERR_LIB_PEM, 107))
+_error_codes.append(("NO_START_LINE", lib.ERR_LIB_PEM, 108))
+_error_codes.append(("PROBLEMS_GETTING_PASSWORD", lib.ERR_LIB_PEM, 109))
+_error_codes.append(("PUBLIC_KEY_NO_RSA", lib.ERR_LIB_PEM, 110))
+_error_codes.append(("PVK_DATA_TOO_SHORT", lib.ERR_LIB_PEM, 124))
+_error_codes.append(("PVK_TOO_SHORT", lib.ERR_LIB_PEM, 125))
+_error_codes.append(("READ_KEY", lib.ERR_LIB_PEM, 111))
+_error_codes.append(("SHORT_HEADER", lib.ERR_LIB_PEM, 112))
+_error_codes.append(("UNSUPPORTED_ENCRYPTION", lib.ERR_LIB_PEM, 114))
+_error_codes.append(("UNSUPPORTED_KEY_COMPONENTS", lib.ERR_LIB_PEM, 126))
+_error_codes.append(("APP_DATA_IN_HANDSHAKE", lib.ERR_LIB_SSL, 100))
+_error_codes.append(("ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", lib.ERR_LIB_SSL, 272))
+_error_codes.append(("BAD_ALERT_RECORD", lib.ERR_LIB_SSL, 101))
+_error_codes.append(("BAD_AUTHENTICATION_TYPE", lib.ERR_LIB_SSL, 102))
+_error_codes.append(("BAD_CHANGE_CIPHER_SPEC", lib.ERR_LIB_SSL, 103))
+_error_codes.append(("BAD_CHECKSUM", lib.ERR_LIB_SSL, 104))
+_error_codes.append(("BAD_DATA", lib.ERR_LIB_SSL, 390))
+_error_codes.append(("BAD_DATA_RETURNED_BY_CALLBACK", lib.ERR_LIB_SSL, 106))
+_error_codes.append(("BAD_DECOMPRESSION", lib.ERR_LIB_SSL, 107))
+_error_codes.append(("BAD_DH_G_LENGTH", lib.ERR_LIB_SSL, 108))
+_error_codes.append(("BAD_DH_PUB_KEY_LENGTH", lib.ERR_LIB_SSL, 109))
+_error_codes.append(("BAD_DH_P_LENGTH", lib.ERR_LIB_SSL, 110))
+_error_codes.append(("BAD_DIGEST_LENGTH", lib.ERR_LIB_SSL, 111))
+_error_codes.append(("BAD_DSA_SIGNATURE", lib.ERR_LIB_SSL, 112))
+_error_codes.append(("BAD_ECC_CERT", lib.ERR_LIB_SSL, 304))
+_error_codes.append(("BAD_ECDSA_SIGNATURE", lib.ERR_LIB_SSL, 305))
+_error_codes.append(("BAD_ECPOINT", lib.ERR_LIB_SSL, 306))
+_error_codes.append(("BAD_HANDSHAKE_LENGTH", lib.ERR_LIB_SSL, 332))
+_error_codes.append(("BAD_HELLO_REQUEST", lib.ERR_LIB_SSL, 105))
+_error_codes.append(("BAD_LENGTH", lib.ERR_LIB_SSL, 271))
+_error_codes.append(("BAD_MAC_DECODE", lib.ERR_LIB_SSL, 113))
+_error_codes.append(("BAD_MAC_LENGTH", lib.ERR_LIB_SSL, 333))
+_error_codes.append(("BAD_MESSAGE_TYPE", lib.ERR_LIB_SSL, 114))
+_error_codes.append(("BAD_PACKET_LENGTH", lib.ERR_LIB_SSL, 115))
+_error_codes.append(("BAD_PROTOCOL_VERSION_NUMBER", lib.ERR_LIB_SSL, 116))
+_error_codes.append(("BAD_PSK_IDENTITY_HINT_LENGTH", lib.ERR_LIB_SSL, 316))
+_error_codes.append(("BAD_RESPONSE_ARGUMENT", lib.ERR_LIB_SSL, 117))
+_error_codes.append(("BAD_RSA_DECRYPT", lib.ERR_LIB_SSL, 118))
+_error_codes.append(("BAD_RSA_ENCRYPT", lib.ERR_LIB_SSL, 119))
+_error_codes.append(("BAD_RSA_E_LENGTH", lib.ERR_LIB_SSL, 120))
+_error_codes.append(("BAD_RSA_MODULUS_LENGTH", lib.ERR_LIB_SSL, 121))
+_error_codes.append(("BAD_RSA_SIGNATURE", lib.ERR_LIB_SSL, 122))
+_error_codes.append(("BAD_SIGNATURE", lib.ERR_LIB_SSL, 123))
+_error_codes.append(("BAD_SRP_A_LENGTH", lib.ERR_LIB_SSL, 347))
+_error_codes.append(("BAD_SRP_B_LENGTH", lib.ERR_LIB_SSL, 348))
+_error_codes.append(("BAD_SRP_G_LENGTH", lib.ERR_LIB_SSL, 349))
+_error_codes.append(("BAD_SRP_N_LENGTH", lib.ERR_LIB_SSL, 350))
+_error_codes.append(("BAD_SRP_PARAMETERS", lib.ERR_LIB_SSL, 371))
+_error_codes.append(("BAD_SRP_S_LENGTH", lib.ERR_LIB_SSL, 351))
+_error_codes.append(("BAD_SRTP_MKI_VALUE", lib.ERR_LIB_SSL, 352))
+_error_codes.append(("BAD_SRTP_PROTECTION_PROFILE_LIST", lib.ERR_LIB_SSL, 353))
+_error_codes.append(("BAD_SSL_FILETYPE", lib.ERR_LIB_SSL, 124))
+_error_codes.append(("BAD_SSL_SESSION_ID_LENGTH", lib.ERR_LIB_SSL, 125))
+_error_codes.append(("BAD_STATE", lib.ERR_LIB_SSL, 126))
+_error_codes.append(("BAD_VALUE", lib.ERR_LIB_SSL, 384))
+_error_codes.append(("BAD_WRITE_RETRY", lib.ERR_LIB_SSL, 127))
+_error_codes.append(("BIO_NOT_SET", lib.ERR_LIB_SSL, 128))
+_error_codes.append(("BLOCK_CIPHER_PAD_IS_WRONG", lib.ERR_LIB_SSL, 129))
+_error_codes.append(("BN_LIB", lib.ERR_LIB_SSL, 130))
+_error_codes.append(("CA_DN_LENGTH_MISMATCH", lib.ERR_LIB_SSL, 131))
+_error_codes.append(("CA_DN_TOO_LONG", lib.ERR_LIB_SSL, 132))
+_error_codes.append(("CA_KEY_TOO_SMALL", lib.ERR_LIB_SSL, 397))
+_error_codes.append(("CA_MD_TOO_WEAK", lib.ERR_LIB_SSL, 398))
+_error_codes.append(("CCS_RECEIVED_EARLY", lib.ERR_LIB_SSL, 133))
+_error_codes.append(("CERTIFICATE_VERIFY_FAILED", lib.ERR_LIB_SSL, 134))
+_error_codes.append(("CERT_CB_ERROR", lib.ERR_LIB_SSL, 377))
+_error_codes.append(("CERT_LENGTH_MISMATCH", lib.ERR_LIB_SSL, 135))
+_error_codes.append(("CHALLENGE_IS_DIFFERENT", lib.ERR_LIB_SSL, 136))
+_error_codes.append(("CIPHER_CODE_WRONG_LENGTH", lib.ERR_LIB_SSL, 137))
+_error_codes.append(("CIPHER_OR_HASH_UNAVAILABLE", lib.ERR_LIB_SSL, 138))
+_error_codes.append(("CIPHER_TABLE_SRC_ERROR", lib.ERR_LIB_SSL, 139))
+_error_codes.append(("CLIENTHELLO_TLSEXT", lib.ERR_LIB_SSL, 226))
+_error_codes.append(("COMPRESSED_LENGTH_TOO_LONG", lib.ERR_LIB_SSL, 140))
+_error_codes.append(("COMPRESSION_DISABLED", lib.ERR_LIB_SSL, 343))
+_error_codes.append(("COMPRESSION_FAILURE", lib.ERR_LIB_SSL, 141))
+_error_codes.append(("COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", lib.ERR_LIB_SSL, 307))
+_error_codes.append(("COMPRESSION_LIBRARY_ERROR", lib.ERR_LIB_SSL, 142))
+_error_codes.append(("CONNECTION_ID_IS_DIFFERENT", lib.ERR_LIB_SSL, 143))
+_error_codes.append(("CONNECTION_TYPE_NOT_SET", lib.ERR_LIB_SSL, 144))
+_error_codes.append(("COOKIE_MISMATCH", lib.ERR_LIB_SSL, 308))
+_error_codes.append(("DATA_BETWEEN_CCS_AND_FINISHED", lib.ERR_LIB_SSL, 145))
+_error_codes.append(("DATA_LENGTH_TOO_LONG", lib.ERR_LIB_SSL, 146))
+_error_codes.append(("DECRYPTION_FAILED", lib.ERR_LIB_SSL, 147))
+_error_codes.append(("DECRYPTION_FAILED_OR_BAD_RECORD_MAC", lib.ERR_LIB_SSL, 281))
+_error_codes.append(("DH_KEY_TOO_SMALL", lib.ERR_LIB_SSL, 372))
+_error_codes.append(("DH_PUBLIC_VALUE_LENGTH_IS_WRONG", lib.ERR_LIB_SSL, 148))
+_error_codes.append(("DIGEST_CHECK_FAILED", lib.ERR_LIB_SSL, 149))
+_error_codes.append(("DTLS_MESSAGE_TOO_BIG", lib.ERR_LIB_SSL, 334))
+_error_codes.append(("DUPLICATE_COMPRESSION_ID", lib.ERR_LIB_SSL, 309))
+_error_codes.append(("ECC_CERT_NOT_FOR_KEY_AGREEMENT", lib.ERR_LIB_SSL, 317))
+_error_codes.append(("ECC_CERT_NOT_FOR_SIGNING", lib.ERR_LIB_SSL, 318))
+_error_codes.append(("ECC_CERT_SHOULD_HAVE_RSA_SIGNATURE", lib.ERR_LIB_SSL, 322))
+_error_codes.append(("ECC_CERT_SHOULD_HAVE_SHA1_SIGNATURE", lib.ERR_LIB_SSL, 323))
+_error_codes.append(("ECDH_REQUIRED_FOR_SUITEB_MODE", lib.ERR_LIB_SSL, 374))
+_error_codes.append(("ECGROUP_TOO_LARGE_FOR_CIPHER", lib.ERR_LIB_SSL, 310))
+_error_codes.append(("EE_KEY_TOO_SMALL", lib.ERR_LIB_SSL, 399))
+_error_codes.append(("EMPTY_SRTP_PROTECTION_PROFILE_LIST", lib.ERR_LIB_SSL, 354))
+_error_codes.append(("ENCRYPTED_LENGTH_TOO_LONG", lib.ERR_LIB_SSL, 150))
+_error_codes.append(("ERROR_GENERATING_TMP_RSA_KEY", lib.ERR_LIB_SSL, 282))
+_error_codes.append(("ERROR_IN_RECEIVED_CIPHER_LIST", lib.ERR_LIB_SSL, 151))
+_error_codes.append(("EXCESSIVE_MESSAGE_SIZE", lib.ERR_LIB_SSL, 152))
+_error_codes.append(("EXTRA_DATA_IN_MESSAGE", lib.ERR_LIB_SSL, 153))
+_error_codes.append(("GOT_A_FIN_BEFORE_A_CCS", lib.ERR_LIB_SSL, 154))
+_error_codes.append(("GOT_NEXT_PROTO_BEFORE_A_CCS", lib.ERR_LIB_SSL, 355))
+_error_codes.append(("GOT_NEXT_PROTO_WITHOUT_EXTENSION", lib.ERR_LIB_SSL, 356))
+_error_codes.append(("HTTPS_PROXY_REQUEST", lib.ERR_LIB_SSL, 155))
+_error_codes.append(("HTTP_REQUEST", lib.ERR_LIB_SSL, 156))
+_error_codes.append(("ILLEGAL_PADDING", lib.ERR_LIB_SSL, 283))
+_error_codes.append(("ILLEGAL_SUITEB_DIGEST", lib.ERR_LIB_SSL, 380))
+_error_codes.append(("INAPPROPRIATE_FALLBACK", lib.ERR_LIB_SSL, 373))
+_error_codes.append(("INCONSISTENT_COMPRESSION", lib.ERR_LIB_SSL, 340))
+_error_codes.append(("INVALID_CHALLENGE_LENGTH", lib.ERR_LIB_SSL, 158))
+_error_codes.append(("INVALID_COMMAND", lib.ERR_LIB_SSL, 280))
+_error_codes.append(("INVALID_COMPRESSION_ALGORITHM", lib.ERR_LIB_SSL, 341))
+_error_codes.append(("INVALID_NULL_CMD_NAME", lib.ERR_LIB_SSL, 385))
+_error_codes.append(("INVALID_PURPOSE", lib.ERR_LIB_SSL, 278))
+_error_codes.append(("INVALID_SERVERINFO_DATA", lib.ERR_LIB_SSL, 388))
+_error_codes.append(("INVALID_SRP_USERNAME", lib.ERR_LIB_SSL, 357))
+_error_codes.append(("INVALID_STATUS_RESPONSE", lib.ERR_LIB_SSL, 328))
+_error_codes.append(("INVALID_TICKET_KEYS_LENGTH", lib.ERR_LIB_SSL, 325))
+_error_codes.append(("KEY_ARG_TOO_LONG", lib.ERR_LIB_SSL, 284))
+_error_codes.append(("KRB5", lib.ERR_LIB_SSL, 285))
+_error_codes.append(("KRB5_C_CC_PRINC", lib.ERR_LIB_SSL, 286))
+_error_codes.append(("KRB5_C_GET_CRED", lib.ERR_LIB_SSL, 287))
+_error_codes.append(("KRB5_C_INIT", lib.ERR_LIB_SSL, 288))
+_error_codes.append(("KRB5_C_MK_REQ", lib.ERR_LIB_SSL, 289))
+_error_codes.append(("KRB5_S_BAD_TICKET", lib.ERR_LIB_SSL, 290))
+_error_codes.append(("KRB5_S_INIT", lib.ERR_LIB_SSL, 291))
+_error_codes.append(("KRB5_S_RD_REQ", lib.ERR_LIB_SSL, 292))
+_error_codes.append(("KRB5_S_TKT_EXPIRED", lib.ERR_LIB_SSL, 293))
+_error_codes.append(("KRB5_S_TKT_NYV", lib.ERR_LIB_SSL, 294))
+_error_codes.append(("KRB5_S_TKT_SKEW", lib.ERR_LIB_SSL, 295))
+_error_codes.append(("LENGTH_MISMATCH", lib.ERR_LIB_SSL, 159))
+_error_codes.append(("LENGTH_TOO_SHORT", lib.ERR_LIB_SSL, 160))
+_error_codes.append(("LIBRARY_BUG", lib.ERR_LIB_SSL, 274))
+_error_codes.append(("LIBRARY_HAS_NO_CIPHERS", lib.ERR_LIB_SSL, 161))
+_error_codes.append(("MESSAGE_TOO_LONG", lib.ERR_LIB_SSL, 296))
+_error_codes.append(("MISSING_DH_DSA_CERT", lib.ERR_LIB_SSL, 162))
+_error_codes.append(("MISSING_DH_KEY", lib.ERR_LIB_SSL, 163))
+_error_codes.append(("MISSING_DH_RSA_CERT", lib.ERR_LIB_SSL, 164))
+_error_codes.append(("MISSING_DSA_SIGNING_CERT", lib.ERR_LIB_SSL, 165))
+_error_codes.append(("MISSING_ECDH_CERT", lib.ERR_LIB_SSL, 382))
+_error_codes.append(("MISSING_ECDSA_SIGNING_CERT", lib.ERR_LIB_SSL, 381))
+_error_codes.append(("MISSING_EXPORT_TMP_DH_KEY", lib.ERR_LIB_SSL, 166))
+_error_codes.append(("MISSING_EXPORT_TMP_RSA_KEY", lib.ERR_LIB_SSL, 167))
+_error_codes.append(("MISSING_RSA_CERTIFICATE", lib.ERR_LIB_SSL, 168))
+_error_codes.append(("MISSING_RSA_ENCRYPTING_CERT", lib.ERR_LIB_SSL, 169))
+_error_codes.append(("MISSING_RSA_SIGNING_CERT", lib.ERR_LIB_SSL, 170))
+_error_codes.append(("MISSING_SRP_PARAM", lib.ERR_LIB_SSL, 358))
+_error_codes.append(("MISSING_TMP_DH_KEY", lib.ERR_LIB_SSL, 171))
+_error_codes.append(("MISSING_TMP_ECDH_KEY", lib.ERR_LIB_SSL, 311))
+_error_codes.append(("MISSING_TMP_RSA_KEY", lib.ERR_LIB_SSL, 172))
+_error_codes.append(("MISSING_TMP_RSA_PKEY", lib.ERR_LIB_SSL, 173))
+_error_codes.append(("MISSING_VERIFY_MESSAGE", lib.ERR_LIB_SSL, 174))
+_error_codes.append(("MULTIPLE_SGC_RESTARTS", lib.ERR_LIB_SSL, 346))
+_error_codes.append(("NON_SSLV2_INITIAL_PACKET", lib.ERR_LIB_SSL, 175))
+_error_codes.append(("NO_CERTIFICATES_RETURNED", lib.ERR_LIB_SSL, 176))
+_error_codes.append(("NO_CERTIFICATE_ASSIGNED", lib.ERR_LIB_SSL, 177))
+_error_codes.append(("NO_CERTIFICATE_RETURNED", lib.ERR_LIB_SSL, 178))
+_error_codes.append(("NO_CERTIFICATE_SET", lib.ERR_LIB_SSL, 179))
+_error_codes.append(("NO_CERTIFICATE_SPECIFIED", lib.ERR_LIB_SSL, 180))
+_error_codes.append(("NO_CIPHERS_AVAILABLE", lib.ERR_LIB_SSL, 181))
+_error_codes.append(("NO_CIPHERS_PASSED", lib.ERR_LIB_SSL, 182))
+_error_codes.append(("NO_CIPHERS_SPECIFIED", lib.ERR_LIB_SSL, 183))
+_error_codes.append(("NO_CIPHER_LIST", lib.ERR_LIB_SSL, 184))
+_error_codes.append(("NO_CIPHER_MATCH", lib.ERR_LIB_SSL, 185))
+_error_codes.append(("NO_CLIENT_CERT_METHOD", lib.ERR_LIB_SSL, 331))
+_error_codes.append(("NO_CLIENT_CERT_RECEIVED", lib.ERR_LIB_SSL, 186))
+_error_codes.append(("NO_COMPRESSION_SPECIFIED", lib.ERR_LIB_SSL, 187))
+_error_codes.append(("NO_GOST_CERTIFICATE_SENT_BY_PEER", lib.ERR_LIB_SSL, 330))
+_error_codes.append(("NO_METHOD_SPECIFIED", lib.ERR_LIB_SSL, 188))
+_error_codes.append(("NO_PEM_EXTENSIONS", lib.ERR_LIB_SSL, 389))
+_error_codes.append(("NO_PRIVATEKEY", lib.ERR_LIB_SSL, 189))
+_error_codes.append(("NO_PRIVATE_KEY_ASSIGNED", lib.ERR_LIB_SSL, 190))
+_error_codes.append(("NO_PROTOCOLS_AVAILABLE", lib.ERR_LIB_SSL, 191))
+_error_codes.append(("NO_PUBLICKEY", lib.ERR_LIB_SSL, 192))
+_error_codes.append(("NO_RENEGOTIATION", lib.ERR_LIB_SSL, 339))
+_error_codes.append(("NO_REQUIRED_DIGEST", lib.ERR_LIB_SSL, 324))
+_error_codes.append(("NO_SHARED_CIPHER", lib.ERR_LIB_SSL, 193))
+_error_codes.append(("NO_SHARED_SIGATURE_ALGORITHMS", lib.ERR_LIB_SSL, 376))
+_error_codes.append(("NO_SRTP_PROFILES", lib.ERR_LIB_SSL, 359))
+_error_codes.append(("NO_VERIFY_CALLBACK", lib.ERR_LIB_SSL, 194))
+_error_codes.append(("NULL_SSL_CTX", lib.ERR_LIB_SSL, 195))
+_error_codes.append(("NULL_SSL_METHOD_PASSED", lib.ERR_LIB_SSL, 196))
+_error_codes.append(("OLD_SESSION_CIPHER_NOT_RETURNED", lib.ERR_LIB_SSL, 197))
+_error_codes.append(("OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", lib.ERR_LIB_SSL, 344))
+_error_codes.append(("ONLY_DTLS_1_2_ALLOWED_IN_SUITEB_MODE", lib.ERR_LIB_SSL, 387))
+_error_codes.append(("ONLY_TLS_1_2_ALLOWED_IN_SUITEB_MODE", lib.ERR_LIB_SSL, 379))
+_error_codes.append(("ONLY_TLS_ALLOWED_IN_FIPS_MODE", lib.ERR_LIB_SSL, 297))
+_error_codes.append(("OPAQUE_PRF_INPUT_TOO_LONG", lib.ERR_LIB_SSL, 327))
+_error_codes.append(("PACKET_LENGTH_TOO_LONG", lib.ERR_LIB_SSL, 198))
+_error_codes.append(("PARSE_TLSEXT", lib.ERR_LIB_SSL, 227))
+_error_codes.append(("PATH_TOO_LONG", lib.ERR_LIB_SSL, 270))
+_error_codes.append(("PEER_DID_NOT_RETURN_A_CERTIFICATE", lib.ERR_LIB_SSL, 199))
+_error_codes.append(("PEER_ERROR", lib.ERR_LIB_SSL, 200))
+_error_codes.append(("PEER_ERROR_CERTIFICATE", lib.ERR_LIB_SSL, 201))
+_error_codes.append(("PEER_ERROR_NO_CERTIFICATE", lib.ERR_LIB_SSL, 202))
+_error_codes.append(("PEER_ERROR_NO_CIPHER", lib.ERR_LIB_SSL, 203))
+_error_codes.append(("PEER_ERROR_UNSUPPORTED_CERTIFICATE_TYPE", lib.ERR_LIB_SSL, 204))
+_error_codes.append(("PEM_NAME_BAD_PREFIX", lib.ERR_LIB_SSL, 391))
+_error_codes.append(("PEM_NAME_TOO_SHORT", lib.ERR_LIB_SSL, 392))
+_error_codes.append(("PRE_MAC_LENGTH_TOO_LONG", lib.ERR_LIB_SSL, 205))
+_error_codes.append(("PROBLEMS_MAPPING_CIPHER_FUNCTIONS", lib.ERR_LIB_SSL, 206))
+_error_codes.append(("PROTOCOL_IS_SHUTDOWN", lib.ERR_LIB_SSL, 207))
+_error_codes.append(("PSK_IDENTITY_NOT_FOUND", lib.ERR_LIB_SSL, 223))
+_error_codes.append(("PSK_NO_CLIENT_CB", lib.ERR_LIB_SSL, 224))
+_error_codes.append(("PSK_NO_SERVER_CB", lib.ERR_LIB_SSL, 225))
+_error_codes.append(("PUBLIC_KEY_ENCRYPT_ERROR", lib.ERR_LIB_SSL, 208))
+_error_codes.append(("PUBLIC_KEY_IS_NOT_RSA", lib.ERR_LIB_SSL, 209))
+_error_codes.append(("PUBLIC_KEY_NOT_RSA", lib.ERR_LIB_SSL, 210))
+_error_codes.append(("READ_BIO_NOT_SET", lib.ERR_LIB_SSL, 211))
+_error_codes.append(("READ_TIMEOUT_EXPIRED", lib.ERR_LIB_SSL, 312))
+_error_codes.append(("READ_WRONG_PACKET_TYPE", lib.ERR_LIB_SSL, 212))
+_error_codes.append(("RECORD_LENGTH_MISMATCH", lib.ERR_LIB_SSL, 213))
+_error_codes.append(("RECORD_TOO_LARGE", lib.ERR_LIB_SSL, 214))
+_error_codes.append(("RECORD_TOO_SMALL", lib.ERR_LIB_SSL, 298))
+_error_codes.append(("RENEGOTIATE_EXT_TOO_LONG", lib.ERR_LIB_SSL, 335))
+_error_codes.append(("RENEGOTIATION_ENCODING_ERR", lib.ERR_LIB_SSL, 336))
+_error_codes.append(("RENEGOTIATION_MISMATCH", lib.ERR_LIB_SSL, 337))
+_error_codes.append(("REQUIRED_CIPHER_MISSING", lib.ERR_LIB_SSL, 215))
+_error_codes.append(("REQUIRED_COMPRESSSION_ALGORITHM_MISSING", lib.ERR_LIB_SSL, 342))
+_error_codes.append(("REUSE_CERT_LENGTH_NOT_ZERO", lib.ERR_LIB_SSL, 216))
+_error_codes.append(("REUSE_CERT_TYPE_NOT_ZERO", lib.ERR_LIB_SSL, 217))
+_error_codes.append(("REUSE_CIPHER_LIST_NOT_ZERO", lib.ERR_LIB_SSL, 218))
+_error_codes.append(("SCSV_RECEIVED_WHEN_RENEGOTIATING", lib.ERR_LIB_SSL, 345))
+_error_codes.append(("SERVERHELLO_TLSEXT", lib.ERR_LIB_SSL, 275))
+_error_codes.append(("SESSION_ID_CONTEXT_UNINITIALIZED", lib.ERR_LIB_SSL, 277))
+_error_codes.append(("SHORT_READ", lib.ERR_LIB_SSL, 219))
+_error_codes.append(("SIGNATURE_ALGORITHMS_ERROR", lib.ERR_LIB_SSL, 360))
+_error_codes.append(("SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", lib.ERR_LIB_SSL, 220))
+_error_codes.append(("SRP_A_CALC", lib.ERR_LIB_SSL, 361))
+_error_codes.append(("SRTP_COULD_NOT_ALLOCATE_PROFILES", lib.ERR_LIB_SSL, 362))
+_error_codes.append(("SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", lib.ERR_LIB_SSL, 363))
+_error_codes.append(("SRTP_UNKNOWN_PROTECTION_PROFILE", lib.ERR_LIB_SSL, 364))
+_error_codes.append(("SSL23_DOING_SESSION_ID_REUSE", lib.ERR_LIB_SSL, 221))
+_error_codes.append(("SSL2_CONNECTION_ID_TOO_LONG", lib.ERR_LIB_SSL, 299))
+_error_codes.append(("SSL3_EXT_INVALID_ECPOINTFORMAT", lib.ERR_LIB_SSL, 321))
+_error_codes.append(("SSL3_EXT_INVALID_SERVERNAME", lib.ERR_LIB_SSL, 319))
+_error_codes.append(("SSL3_EXT_INVALID_SERVERNAME_TYPE", lib.ERR_LIB_SSL, 320))
+_error_codes.append(("SSL3_SESSION_ID_TOO_LONG", lib.ERR_LIB_SSL, 300))
+_error_codes.append(("SSL3_SESSION_ID_TOO_SHORT", lib.ERR_LIB_SSL, 222))
+_error_codes.append(("SSLV3_ALERT_BAD_CERTIFICATE", lib.ERR_LIB_SSL, 1042))
+_error_codes.append(("SSLV3_ALERT_BAD_RECORD_MAC", lib.ERR_LIB_SSL, 1020))
+_error_codes.append(("SSLV3_ALERT_CERTIFICATE_EXPIRED", lib.ERR_LIB_SSL, 1045))
+_error_codes.append(("SSLV3_ALERT_CERTIFICATE_REVOKED", lib.ERR_LIB_SSL, 1044))
+_error_codes.append(("SSLV3_ALERT_CERTIFICATE_UNKNOWN", lib.ERR_LIB_SSL, 1046))
+_error_codes.append(("SSLV3_ALERT_DECOMPRESSION_FAILURE", lib.ERR_LIB_SSL, 1030))
+_error_codes.append(("SSLV3_ALERT_HANDSHAKE_FAILURE", lib.ERR_LIB_SSL, 1040))
+_error_codes.append(("SSLV3_ALERT_ILLEGAL_PARAMETER", lib.ERR_LIB_SSL, 1047))
+_error_codes.append(("SSLV3_ALERT_NO_CERTIFICATE", lib.ERR_LIB_SSL, 1041))
+_error_codes.append(("SSLV3_ALERT_UNEXPECTED_MESSAGE", lib.ERR_LIB_SSL, 1010))
+_error_codes.append(("SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", lib.ERR_LIB_SSL, 1043))
+_error_codes.append(("SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", lib.ERR_LIB_SSL, 228))
+_error_codes.append(("SSL_HANDSHAKE_FAILURE", lib.ERR_LIB_SSL, 229))
+_error_codes.append(("SSL_LIBRARY_HAS_NO_CIPHERS", lib.ERR_LIB_SSL, 230))
+_error_codes.append(("SSL_NEGATIVE_LENGTH", lib.ERR_LIB_SSL, 372))
+_error_codes.append(("SSL_SESSION_ID_CALLBACK_FAILED", lib.ERR_LIB_SSL, 301))
+_error_codes.append(("SSL_SESSION_ID_CONFLICT", lib.ERR_LIB_SSL, 302))
+_error_codes.append(("SSL_SESSION_ID_CONTEXT_TOO_LONG", lib.ERR_LIB_SSL, 273))
+_error_codes.append(("SSL_SESSION_ID_HAS_BAD_LENGTH", lib.ERR_LIB_SSL, 303))
+_error_codes.append(("SSL_SESSION_ID_IS_DIFFERENT", lib.ERR_LIB_SSL, 231))
+_error_codes.append(("TLSV1_ALERT_ACCESS_DENIED", lib.ERR_LIB_SSL, 1049))
+_error_codes.append(("TLSV1_ALERT_DECODE_ERROR", lib.ERR_LIB_SSL, 1050))
+_error_codes.append(("TLSV1_ALERT_DECRYPTION_FAILED", lib.ERR_LIB_SSL, 1021))
+_error_codes.append(("TLSV1_ALERT_DECRYPT_ERROR", lib.ERR_LIB_SSL, 1051))
+_error_codes.append(("TLSV1_ALERT_EXPORT_RESTRICTION", lib.ERR_LIB_SSL, 1060))
+_error_codes.append(("TLSV1_ALERT_INAPPROPRIATE_FALLBACK", lib.ERR_LIB_SSL, 1086))
+_error_codes.append(("TLSV1_ALERT_INSUFFICIENT_SECURITY", lib.ERR_LIB_SSL, 1071))
+_error_codes.append(("TLSV1_ALERT_INTERNAL_ERROR", lib.ERR_LIB_SSL, 1080))
+_error_codes.append(("TLSV1_ALERT_NO_RENEGOTIATION", lib.ERR_LIB_SSL, 1100))
+_error_codes.append(("TLSV1_ALERT_PROTOCOL_VERSION", lib.ERR_LIB_SSL, 1070))
+_error_codes.append(("TLSV1_ALERT_RECORD_OVERFLOW", lib.ERR_LIB_SSL, 1022))
+_error_codes.append(("TLSV1_ALERT_UNKNOWN_CA", lib.ERR_LIB_SSL, 1048))
+_error_codes.append(("TLSV1_ALERT_USER_CANCELLED", lib.ERR_LIB_SSL, 1090))
+_error_codes.append(("TLSV1_BAD_CERTIFICATE_HASH_VALUE", lib.ERR_LIB_SSL, 1114))
+_error_codes.append(("TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", lib.ERR_LIB_SSL, 1113))
+_error_codes.append(("TLSV1_CERTIFICATE_UNOBTAINABLE", lib.ERR_LIB_SSL, 1111))
+_error_codes.append(("TLSV1_UNRECOGNIZED_NAME", lib.ERR_LIB_SSL, 1112))
+_error_codes.append(("TLSV1_UNSUPPORTED_EXTENSION", lib.ERR_LIB_SSL, 1110))
+_error_codes.append(("TLS_CLIENT_CERT_REQ_WITH_ANON_CIPHER", lib.ERR_LIB_SSL, 232))
+_error_codes.append(("TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", lib.ERR_LIB_SSL, 365))
+_error_codes.append(("TLS_HEARTBEAT_PENDING", lib.ERR_LIB_SSL, 366))
+_error_codes.append(("TLS_ILLEGAL_EXPORTER_LABEL", lib.ERR_LIB_SSL, 367))
+_error_codes.append(("TLS_INVALID_ECPOINTFORMAT_LIST", lib.ERR_LIB_SSL, 157))
+_error_codes.append(("TLS_PEER_DID_NOT_RESPOND_WITH_CERTIFICATE_LIST", lib.ERR_LIB_SSL, 233))
+_error_codes.append(("TLS_RSA_ENCRYPTED_VALUE_LENGTH_IS_WRONG", lib.ERR_LIB_SSL, 234))
+_error_codes.append(("TRIED_TO_USE_UNSUPPORTED_CIPHER", lib.ERR_LIB_SSL, 235))
+_error_codes.append(("UNABLE_TO_DECODE_DH_CERTS", lib.ERR_LIB_SSL, 236))
+_error_codes.append(("UNABLE_TO_DECODE_ECDH_CERTS", lib.ERR_LIB_SSL, 313))
+_error_codes.append(("UNABLE_TO_EXTRACT_PUBLIC_KEY", lib.ERR_LIB_SSL, 237))
+_error_codes.append(("UNABLE_TO_FIND_DH_PARAMETERS", lib.ERR_LIB_SSL, 238))
+_error_codes.append(("UNABLE_TO_FIND_ECDH_PARAMETERS", lib.ERR_LIB_SSL, 314))
+_error_codes.append(("UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", lib.ERR_LIB_SSL, 239))
+_error_codes.append(("UNABLE_TO_FIND_SSL_METHOD", lib.ERR_LIB_SSL, 240))
+_error_codes.append(("UNABLE_TO_LOAD_SSL2_MD5_ROUTINES", lib.ERR_LIB_SSL, 241))
+_error_codes.append(("UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", lib.ERR_LIB_SSL, 242))
+_error_codes.append(("UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", lib.ERR_LIB_SSL, 243))
+_error_codes.append(("UNEXPECTED_MESSAGE", lib.ERR_LIB_SSL, 244))
+_error_codes.append(("UNEXPECTED_RECORD", lib.ERR_LIB_SSL, 245))
+_error_codes.append(("UNINITIALIZED", lib.ERR_LIB_SSL, 276))
+_error_codes.append(("UNKNOWN_ALERT_TYPE", lib.ERR_LIB_SSL, 246))
+_error_codes.append(("UNKNOWN_CERTIFICATE_TYPE", lib.ERR_LIB_SSL, 247))
+_error_codes.append(("UNKNOWN_CIPHER_RETURNED", lib.ERR_LIB_SSL, 248))
+_error_codes.append(("UNKNOWN_CIPHER_TYPE", lib.ERR_LIB_SSL, 249))
+_error_codes.append(("UNKNOWN_CMD_NAME", lib.ERR_LIB_SSL, 386))
+_error_codes.append(("UNKNOWN_DIGEST", lib.ERR_LIB_SSL, 368))
+_error_codes.append(("UNKNOWN_KEY_EXCHANGE_TYPE", lib.ERR_LIB_SSL, 250))
+_error_codes.append(("UNKNOWN_PKEY_TYPE", lib.ERR_LIB_SSL, 251))
+_error_codes.append(("UNKNOWN_PROTOCOL", lib.ERR_LIB_SSL, 252))
+_error_codes.append(("UNKNOWN_REMOTE_ERROR_TYPE", lib.ERR_LIB_SSL, 253))
+_error_codes.append(("UNKNOWN_SSL_VERSION", lib.ERR_LIB_SSL, 254))
+_error_codes.append(("UNKNOWN_STATE", lib.ERR_LIB_SSL, 255))
+_error_codes.append(("UNSAFE_LEGACY_RENEGOTIATION_DISABLED", lib.ERR_LIB_SSL, 338))
+_error_codes.append(("UNSUPPORTED_CIPHER", lib.ERR_LIB_SSL, 256))
+_error_codes.append(("UNSUPPORTED_COMPRESSION_ALGORITHM", lib.ERR_LIB_SSL, 257))
+_error_codes.append(("UNSUPPORTED_DIGEST_TYPE", lib.ERR_LIB_SSL, 326))
+_error_codes.append(("UNSUPPORTED_ELLIPTIC_CURVE", lib.ERR_LIB_SSL, 315))
+_error_codes.append(("UNSUPPORTED_PROTOCOL", lib.ERR_LIB_SSL, 258))
+_error_codes.append(("UNSUPPORTED_SSL_VERSION", lib.ERR_LIB_SSL, 259))
+_error_codes.append(("UNSUPPORTED_STATUS_TYPE", lib.ERR_LIB_SSL, 329))
+_error_codes.append(("USE_SRTP_NOT_NEGOTIATED", lib.ERR_LIB_SSL, 369))
+_error_codes.append(("VERSION_TOO_LOW", lib.ERR_LIB_SSL, 396))
+_error_codes.append(("WRITE_BIO_NOT_SET", lib.ERR_LIB_SSL, 260))
+_error_codes.append(("WRONG_CERTIFICATE_TYPE", lib.ERR_LIB_SSL, 383))
+_error_codes.append(("WRONG_CIPHER_RETURNED", lib.ERR_LIB_SSL, 261))
+_error_codes.append(("WRONG_CURVE", lib.ERR_LIB_SSL, 378))
+_error_codes.append(("WRONG_MESSAGE_TYPE", lib.ERR_LIB_SSL, 262))
+_error_codes.append(("WRONG_NUMBER_OF_KEY_BITS", lib.ERR_LIB_SSL, 263))
+_error_codes.append(("WRONG_SIGNATURE_LENGTH", lib.ERR_LIB_SSL, 264))
+_error_codes.append(("WRONG_SIGNATURE_SIZE", lib.ERR_LIB_SSL, 265))
+_error_codes.append(("WRONG_SIGNATURE_TYPE", lib.ERR_LIB_SSL, 370))
+_error_codes.append(("WRONG_SSL_VERSION", lib.ERR_LIB_SSL, 266))
+_error_codes.append(("WRONG_VERSION_NUMBER", lib.ERR_LIB_SSL, 267))
+_error_codes.append(("X509_LIB", lib.ERR_LIB_SSL, 268))
+_error_codes.append(("X509_VERIFICATION_SETUP_PROBLEMS", lib.ERR_LIB_SSL, 269))
+_error_codes.append(("AKID_MISMATCH", lib.ERR_LIB_X509, 110))
+_error_codes.append(("BAD_X509_FILETYPE", lib.ERR_LIB_X509, 100))
+_error_codes.append(("BASE64_DECODE_ERROR", lib.ERR_LIB_X509, 118))
+_error_codes.append(("CANT_CHECK_DH_KEY", lib.ERR_LIB_X509, 114))
+_error_codes.append(("CERT_ALREADY_IN_HASH_TABLE", lib.ERR_LIB_X509, 101))
+_error_codes.append(("CRL_ALREADY_DELTA", lib.ERR_LIB_X509, 127))
+_error_codes.append(("CRL_VERIFY_FAILURE", lib.ERR_LIB_X509, 131))
+_error_codes.append(("ERR_ASN1_LIB", lib.ERR_LIB_X509, 102))
+_error_codes.append(("IDP_MISMATCH", lib.ERR_LIB_X509, 128))
+_error_codes.append(("INVALID_DIRECTORY", lib.ERR_LIB_X509, 113))
+_error_codes.append(("INVALID_FIELD_NAME", lib.ERR_LIB_X509, 119))
+_error_codes.append(("INVALID_TRUST", lib.ERR_LIB_X509, 123))
+_error_codes.append(("ISSUER_MISMATCH", lib.ERR_LIB_X509, 129))
+_error_codes.append(("KEY_TYPE_MISMATCH", lib.ERR_LIB_X509, 115))
+_error_codes.append(("KEY_VALUES_MISMATCH", lib.ERR_LIB_X509, 116))
+_error_codes.append(("LOADING_CERT_DIR", lib.ERR_LIB_X509, 103))
+_error_codes.append(("LOADING_DEFAULTS", lib.ERR_LIB_X509, 104))
+_error_codes.append(("METHOD_NOT_SUPPORTED", lib.ERR_LIB_X509, 124))
+_error_codes.append(("NEWER_CRL_NOT_NEWER", lib.ERR_LIB_X509, 132))
+_error_codes.append(("NO_CERT_SET_FOR_US_TO_VERIFY", lib.ERR_LIB_X509, 105))
+_error_codes.append(("NO_CRL_NUMBER", lib.ERR_LIB_X509, 130))
+_error_codes.append(("PUBLIC_KEY_DECODE_ERROR", lib.ERR_LIB_X509, 125))
+_error_codes.append(("PUBLIC_KEY_ENCODE_ERROR", lib.ERR_LIB_X509, 126))
+_error_codes.append(("SHOULD_RETRY", lib.ERR_LIB_X509, 106))
+_error_codes.append(("UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", lib.ERR_LIB_X509, 107))
+_error_codes.append(("UNABLE_TO_GET_CERTS_PUBLIC_KEY", lib.ERR_LIB_X509, 108))
+_error_codes.append(("UNKNOWN_KEY_TYPE", lib.ERR_LIB_X509, 117))
+_error_codes.append(("UNKNOWN_NID", lib.ERR_LIB_X509, 109))
+_error_codes.append(("UNKNOWN_PURPOSE_ID", lib.ERR_LIB_X509, 121))
+_error_codes.append(("UNKNOWN_TRUST_ID", lib.ERR_LIB_X509, 120))
+_error_codes.append(("UNSUPPORTED_ALGORITHM", lib.ERR_LIB_X509, 111))
+_error_codes.append(("WRONG_LOOKUP_TYPE", lib.ERR_LIB_X509, 112))
+_error_codes.append(("WRONG_TYPE", lib.ERR_LIB_X509, 122))
diff --git a/lib_pypy/openssl/tools/make_ssl_data.py b/lib_pypy/openssl/tools/make_ssl_data.py
new file mode 100755
--- /dev/null
+++ b/lib_pypy/openssl/tools/make_ssl_data.py
@@ -0,0 +1,82 @@
+#! /usr/bin/env python3
+
+"""
+This script should be called *manually* when we want to upgrade SSLError
+`library` and `reason` mnemnonics to a more recent OpenSSL version.
+
+It takes two arguments:
+- the path to the OpenSSL source tree (e.g. git checkout)
+- the path to the C file to be generated
+  (probably Modules/_ssl_data.h)
+"""
+
+import datetime
+import os
+import re
+import sys
+import _ssl
+
+
+def parse_error_codes(h_file, prefix, libcode):
+    pat = re.compile(r"#define\W+(%s([\w]+))\W+(\d+)\b" % re.escape(prefix))
+    codes = []
+    with open(h_file, "r", encoding="latin1") as f:
+        for line in f:
+            match = pat.search(line)
+            if match:
+                code, name, num = match.groups()
+                num = int(num)
+                # e.g. ("SSL_R_BAD_DATA", ("ERR_LIB_SSL", "BAD_DATA", 390))
+                codes.append((code, (libcode, name, num)))
+    return codes
+
+if __name__ == "__main__":
+    openssl_inc = sys.argv[1]
+    outfile = sys.argv[2]
+    use_stdout = outfile == '-'
+    f = sys.stdout if use_stdout else open(outfile, "w")
+    error_libraries = {
+        # mnemonic -> (library code, error prefix, header file)
+        'PEM': ('ERR_LIB_PEM', 'PEM_R_', 'include/openssl/pem.h'),
+        'SSL': ('ERR_LIB_SSL', 'SSL_R_', 'include/openssl/ssl.h'),
+        'X509': ('ERR_LIB_X509', 'X509_R_', 'include/openssl/x509.h'),
+    }
+
+    # Read codes from libraries
+    new_codes = []
+    for libcode, prefix, h_file in sorted(error_libraries.values()):
+        new_codes += parse_error_codes(os.path.join(openssl_inc, h_file),
+                                       prefix, libcode)
+    new_code_nums = set((libcode, num)
+                        for (code, (libcode, name, num)) in new_codes)
+
+    # Merge with existing codes (in case some old codes disappeared).
+    codes = {}
+    for errname, (libnum, errnum) in _ssl.err_names_to_codes.items():
+        lib = error_libraries[_ssl.lib_codes_to_names[libnum]]
+        libcode = lib[0]              # e.g. ERR_LIB_PEM
+        errcode = lib[1] + errname    # e.g. SSL_R_BAD_SSL_SESSION_ID_LENGTH
+        # Only keep it if the numeric codes weren't reused
+        if (libcode, errnum) not in new_code_nums:
+            codes[errcode] = libcode, errname, errnum
+    codes.update(dict(new_codes))
+
+    def w(l):
+        f.write(l + "\n")
+    w("# File generated by tools/make_ssl_data.py")
+    w("# Generated on %s" % datetime.datetime.now().isoformat())
+    w("")
+    w("from _openssl import ffi, lib ")
+
+    w("_lib_codes = []")
+    for mnemo, (libcode, _, _) in sorted(error_libraries.items()):
+        w('_lib_codes.append(("%s", lib.%s))' % (mnemo, libcode))
+
+    w("_error_codes = []")
+    for errcode, (libcode, name, num) in sorted(codes.items()):
+        #w('if lib.%s:' % (errcode))
+        #w('    _error_codes.append(("%s", lib.%s, lib.%s))' % (name, libcode, errcode))
+        #w('else:')
+        w('_error_codes.append(("%s", lib.%s, %d))' % (name, libcode, num))
+    if not use_stdout:
+        f.close()

From pypy.commits at gmail.com  Wed Nov 16 11:37:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Nov 2016 08:37:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: (arigo,
 cfbolz): continue refactoring
Message-ID: <582c8b41.8b9a1c0a.f0db4.9c2d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88408:b4bb834f45e9
Date: 2016-11-16 16:47 +0000
http://bitbucket.org/pypy/pypy/changeset/b4bb834f45e9/

Log:	(arigo, cfbolz): continue refactoring

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -622,8 +622,8 @@
 
     ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP_START: 1,
-    ops.WITH_CLEANUP_FINISH: -2,
+    ops.WITH_CLEANUP_START: 0,
+    ops.WITH_CLEANUP_FINISH: -1,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
     ops.POP_EXCEPT: -1,
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -226,7 +226,6 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
-    @jit.dont_look_inside
     def sys_exc_info(self):
         """Implements sys.exc_info().
         Return an OperationError instance or None.
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -22,6 +22,7 @@
         self._qualname = qualname   # may be null, use get_qualname()
         if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
             self.register_finalizer(self.space)
+        self.saved_operr = None
 
     def get_name(self):
         # 'name' is a byte string that is valid utf-8
@@ -46,6 +47,7 @@
                            unicode(addrstring)))
 
     def descr__reduce__(self, space):
+        # DEAD CODE, see frame.__reduce__
         from pypy.interpreter.mixedmodule import MixedModule
         w_mod = space.getbuiltinmodule('_pickle_support')
         mod = space.interp_w(MixedModule, w_mod)
@@ -61,6 +63,7 @@
                                space.newtuple(tup)])
 
     def descr__setstate__(self, space, w_args):
+        # DEAD CODE, see frame.__reduce__
         from rpython.rlib.objectmodel import instantiate
         args_w = space.unpackiterable(w_args)
         w_framestate, w_running = args_w
@@ -136,6 +139,10 @@
         space = self.space
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
+        ec = space.getexecutioncontext()
+        current_exc_info = ec.sys_exc_info()
+        if self.saved_operr is not None:
+            ec.set_sys_exc_info(self.saved_operr)
         self.running = True
         try:
             w_result = frame.execute_frame(self, w_arg_or_err)
@@ -150,6 +157,8 @@
         finally:
             frame.f_backref = jit.vref_None
             self.running = False
+            self.saved_operr = ec.sys_exc_info()
+            ec.set_sys_exc_info(current_exc_info)
         return w_result
 
     def resume_execute_frame(self, frame, w_arg_or_err):
@@ -162,7 +171,7 @@
             try:
                 self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
-                operr.record_context(space, frame)
+                operr.record_context(space, space.getexecutioncontext())
                 return frame.handle_generator_error(operr)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
@@ -226,7 +235,7 @@
                                 space.wrap("%s raised StopIteration" %
                                            self.KIND))
             e2.chain_exceptions(space, e)
-            e2.record_context(space, self.frame)
+            e2.record_context(space, space.getexecutioncontext())
             raise e2
         else:
             space.warn(space.wrap(u"generator '%s' raised StopIteration"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -457,6 +457,9 @@
 
     @jit.dont_look_inside
     def descr__reduce__(self, space):
+        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
+        # Either re-add at some point, or kill this code.
+        dead_code
         from pypy.interpreter.mixedmodule import MixedModule
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
@@ -467,6 +470,8 @@
 
     @jit.dont_look_inside
     def _reduce_state(self, space):
+        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
+        dead_code
         from pypy.module._pickle_support import maker # helper fns
         w = space.wrap
         nt = space.newtuple
@@ -515,6 +520,9 @@
 
     @jit.dont_look_inside
     def descr__setstate__(self, space, w_args):
+        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
+        # Either re-add at some point, or kill this code.
+        dead_code
         from pypy.module._pickle_support import maker # helper fns
         from pypy.interpreter.pycode import PyCode
         from pypy.interpreter.module import Module
@@ -882,23 +890,6 @@
             return space.wrap(self.builtin is not space.builtin)
         return space.w_False
 
-    @jit.unroll_safe
-    @specialize.arg(2)
-    def _exc_info_unroll(self, space, for_hidden=False):
-        """Return the most recent OperationError being handled in the
-        call stack
-        """
-        frame = self
-        while frame:
-            last = frame.last_exception
-            if last is not None:
-                if last is get_cleared_operation_error(self.space):
-                    break
-                if for_hidden or not frame.hide():
-                    return last
-            frame = frame.f_backref()
-        return None
-
     def get_generator(self):
         if self.space.config.translation.rweakref:
             return self.f_generator_wref()
@@ -906,10 +897,9 @@
             return self.f_generator_nowref
 
     def descr_clear(self, space):
+        """F.clear(): clear most references held by the frame"""
         # Clears a random subset of the attributes (e.g. the fast
-        # locals, but not f_locals).  Also clears last_exception, which
-        # is not quite like CPython when it clears f_exc_* (however
-        # there might not be an observable difference).
+        # locals, but not f_locals).
         if not self.frame_finished_execution:
             if not self._is_generator_or_coroutine():
                 raise oefmt(space.w_RuntimeError,
@@ -923,7 +913,6 @@
                 # awaited" in this case too.  Does it make any sense?
                 gen.descr_close()
 
-        self.last_exception = None
         debug = self.getdebug()
         if debug is not None:
             debug.w_f_trace = None
@@ -937,6 +926,7 @@
                 w_newvalue = None
             self.locals_cells_stack_w[i] = w_newvalue
         self.valuestackdepth = 0
+        self.lastblock = None    # the FrameBlock chained list
 
     def _convert_unexpected_exception(self, e):
         from pypy.interpreter import error
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -63,10 +63,7 @@
         try:
             while True:
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
-        except Return:
-            self.last_exception = None
-            return self.popvalue()
-        except Yield:
+        except ExitFrame:
             return self.popvalue()
 
     def handle_bytecode(self, co_code, next_instr, ec):
@@ -746,11 +743,13 @@
             self.setdictscope(w_locals)
 
     def POP_EXCEPT(self, oparg, next_instr):
-        pass    # no-op for now: done by the END_FINALLY that follows anyway
+        block = self.pop_block()
+        assert isinstance(block, SysExcInfoRestorer)
+        block.cleanupstack(self)   # restores ec.sys_exc_operror
 
     def POP_BLOCK(self, oparg, next_instr):
         block = self.pop_block()
-        block.cleanup(self)  # the block knows how to clean up the value stack
+        block.pop_block(self)  # the block knows how to clean up the value stack
 
     def save_and_change_sys_exc_info(self, operationerr):
         ec = self.space.getexecutioncontext()
@@ -774,7 +773,7 @@
 
         block = self.pop_block()
         assert isinstance(block, SysExcInfoRestorer)
-        block.cleanupstack()   # restores ec.sys_exc_operror
+        block.cleanupstack(self)   # restores ec.sys_exc_operror
 
         w_top = self.popvalue()
         if self.space.is_w(w_top, self.space.w_None):
@@ -1161,41 +1160,28 @@
         if isinstance(w_unroller, SApplicationException):
             # app-level exception
             operr = w_unroller.operr
-            # this looks again like the kind of code we have. except that it's a call, not a block
-            old_last_exception = self.last_exception
-            self.last_exception = operr
             w_traceback = self.space.wrap(operr.get_traceback())
-            try:
-                w_res = self.call_contextmanager_exit_function(
-                    w_exitfunc,
-                    operr.w_type,
-                    operr.get_w_value(self.space),
-                    w_traceback)
-            except:
-                self.last_exception = old_last_exception
-                raise
-            # push a marker that also contains the old_last_exception,
-            # which must be restored as 'self.last_exception' but only
-            # in WITH_CLEANUP_FINISH
-            self.pushvalue(SApplicationException(old_last_exception))
+            w_res = self.call_contextmanager_exit_function(
+                w_exitfunc,
+                operr.w_type,
+                operr.get_w_value(self.space),
+                w_traceback)
         else:
             w_res = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 self.space.w_None,
                 self.space.w_None,
                 self.space.w_None)
-            self.pushvalue(self.space.w_None)
         self.pushvalue(w_res)
+        # in the stack now:  [w_res, w_unroller-or-w_None..]
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        w_marker = self.popvalue()
-        if isinstance(w_marker, SApplicationException):
-            self.last_exception = w_marker.operr    # may be None
-            if self.space.is_true(w_suppress):
-                # __exit__() returned True -> Swallow the exception.
-                self.settopvalue(self.space.w_None)
+        if self.space.is_true(w_suppress):
+            # __exit__() returned True -> Swallow the exception.
+            self.settopvalue(self.space.w_None)
         # this is always followed by END_FINALLY
+        # in the stack now: [w_unroller-or-w_None..]
 
     @jit.unroll_safe
     def call_function(self, oparg, w_starstar=None, has_vararg=False):
@@ -1476,8 +1462,8 @@
 
     def SETUP_ASYNC_WITH(self, offsettoend, next_instr):
         res = self.popvalue()
-        block = WithBlock(self.valuestackdepth,
-                          next_instr + offsettoend, self.lastblock)
+        block = FinallyBlock(self.valuestackdepth,
+                             next_instr + offsettoend, self.lastblock)
         self.lastblock = block
         self.pushvalue(res)
 
@@ -1559,12 +1545,15 @@
 
 ### ____________________________________________________________ ###
 
+class ExitFrame(Exception):
+    pass
 
-class Return(Exception):
+
+class Return(ExitFrame):
     """Raised when exiting a frame via a 'return' statement."""
 
 
-class Yield(Exception):
+class Yield(ExitFrame):
     """Raised when exiting a frame via a 'yield' statement."""
 
 
@@ -1656,7 +1645,7 @@
     def cleanupstack(self, frame):
         frame.dropvaluesuntil(self.valuestackdepth)
 
-    def cleanup(self, frame):
+    def pop_block(self, frame):
         "Clean up a frame when we normally exit the block."
         self.cleanupstack(frame)
 
@@ -1707,6 +1696,9 @@
         self.operr = operr
         self.previous = previous
 
+    def pop_block(self, frame):
+        assert False # never called
+
     def handle(self, frame, unroller):
         assert False # never called
 
@@ -1763,6 +1755,10 @@
         frame.save_and_change_sys_exc_info(operationerr)
         return r_uint(self.handlerposition)   # jump to the handler
 
+    def pop_block(self, frame):
+        self.cleanupstack(frame)
+        frame.save_and_change_sys_exc_info(None)
+
 
 block_classes = {'SYS_EXC_INFO_RESTORER': SysExcInfoRestorer,
                  'SETUP_LOOP': LoopBlock,
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -2,9 +2,8 @@
 from pypy.interpreter import executioncontext
 from pypy.interpreter.error import OperationError
 
-class Finished(OperationError):
-    def __init__(self):
-        OperationError.__init__(self, "exception_class", "exception_value")
+class Finished(Exception):
+    pass
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -457,6 +457,46 @@
         assert closed == [True]
         """
 
+    def test_exc_info_in_generator(self):
+        import sys
+        def g():
+            try:
+                raise ValueError
+            except ValueError:
+                yield sys.exc_info()[0]
+                yield sys.exc_info()[0]
+        try:
+            raise IndexError
+        except IndexError:
+            gen = g()
+            assert sys.exc_info()[0] is IndexError
+            assert next(gen) is ValueError
+            assert sys.exc_info()[0] is IndexError
+            assert next(gen) is ValueError
+            assert sys.exc_info()[0] is IndexError
+            raises(StopIteration, next, gen)
+            assert sys.exc_info()[0] is IndexError
+
+    def test_exc_info_in_generator_2(self):
+        import sys
+        def g():
+            yield sys.exc_info()[0]
+            try:
+                raise LookupError
+            except LookupError:
+                yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+        try:
+            raise IndexError
+        except IndexError:
+            gen = g()     # the IndexError is not captured at all
+        try:
+            raise ValueError
+        except ValueError:
+            assert next(gen) is ValueError
+            assert next(gen) is LookupError
+            assert next(gen) is ValueError
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -148,30 +148,6 @@
         assert f1bis is f1
         assert f0.f_back is f1
 
-    def test_f_exc_xxx(self):
-        import sys
-
-        class OuterException(Exception):
-            pass
-        class InnerException(Exception):
-            pass
-
-        def g(exc_info):
-            f = sys._getframe()
-            assert f.f_exc_type is None
-            assert f.f_exc_value is None
-            assert f.f_exc_traceback is None
-            try:
-                raise InnerException
-            except:
-                assert f.f_exc_type is exc_info[0]
-                assert f.f_exc_value is exc_info[1]
-                assert f.f_exc_traceback is exc_info[2]
-        try:
-            raise OuterException
-        except:
-            g(sys.exc_info())
-
     def test_virtualref_through_traceback(self):
         import sys
         def g():
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -351,6 +351,7 @@
         assert list(result) == [2,3,4]
 
     def test_pickle_generator(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         import types
         mod = types.ModuleType('mod')
@@ -375,6 +376,7 @@
             del sys.modules['mod']
 
     def test_pickle_generator_blk(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         # same as above but with the generator inside a block
         import types
@@ -448,6 +450,7 @@
 
 
     def test_pickle_generator_crash(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         import pickle
 
@@ -464,7 +467,7 @@
         assert 'finished' in repr(y)
         assert y.gi_code is None
 
-class AppTestGeneratorCloning:
+class XAppTestGeneratorCloning:
 
     def setup_class(cls):
         try:
@@ -528,7 +531,7 @@
             raises(StopIteration, next, g3)
             raises(StopIteration, next, g4)
 
-class AppTestFramePickling(object):
+class XAppTestFramePickling(object):
     pytestmark = py.test.mark.skipif("config.option.runappdirect")
     spaceconfig = {
         "usemodules": ["struct"]
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -607,8 +607,8 @@
 PyCode.typedef.acceptable_as_base_class = False
 
 PyFrame.typedef = TypeDef('frame',
-    __reduce__ = interp2app(PyFrame.descr__reduce__),
-    __setstate__ = interp2app(PyFrame.descr__setstate__),
+    #__reduce__ = interp2app(PyFrame.descr__reduce__),  --- logic not updated
+    #__setstate__ = interp2app(PyFrame.descr__setstate__),
     clear = interp2app(PyFrame.descr_clear),
     f_builtins = GetSetProperty(PyFrame.fget_f_builtins),
     f_lineno = GetSetProperty(PyFrame.fget_f_lineno, PyFrame.fset_f_lineno),
@@ -784,8 +784,8 @@
 
 GeneratorIterator.typedef = TypeDef("generator",
     __repr__   = interp2app(GeneratorIterator.descr__repr__),
-    __reduce__   = interp2app(GeneratorIterator.descr__reduce__),
-    __setstate__ = interp2app(GeneratorIterator.descr__setstate__),
+    #__reduce__   = interp2app(GeneratorIterator.descr__reduce__),
+    #__setstate__ = interp2app(GeneratorIterator.descr__setstate__),
     __next__   = interp2app(GeneratorIterator.descr_next,
                             descrmismatch='__next__'),
     send       = interp2app(GeneratorIterator.descr_send,
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -26,7 +26,6 @@
                            'valuestackdepth',
                            'locals_cells_stack_w[*]',
                            'debugdata',
-                           'last_exception',
                            'lastblock',
                            'w_globals',
                            ]
@@ -98,7 +97,6 @@
             jit.hint(self, force_virtualizable=True)
             return w_result
         except Return:
-            self.last_exception = None
             return self.popvalue()
 
     def jump_absolute(self, jumpto, ec):
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -107,8 +107,7 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
-def exc_info_without_tb(space, frame):
-    operror = frame.last_exception
+def exc_info_without_tb(space, operror):
     return space.newtuple([operror.w_type, operror.get_w_value(space),
                            space.w_None])
 
@@ -152,10 +151,11 @@
                         if space.int_w(w_constant) <= 2:
                             need_all_three_args = False
     #
-    if need_all_three_args or frame.last_exception is None or frame.hide():
+    operror = space.getexecutioncontext().sys_exc_info()
+    if need_all_three_args or operror is None or frame.hide():
         return exc_info_with_tb(space)
     else:
-        return exc_info_without_tb(space, frame)
+        return exc_info_without_tb(space, operror)
 
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -210,26 +210,20 @@
                                space.newtuple([w_BuiltinAssertionError]),
                                w_dict)
 
-def _exc_info(space, err):
-    """Hack the fact that exc_info() isn't set until a app except
-    block catches it."""
-    err.normalize_exception(space)
-    frame = space.getexecutioncontext().gettopframe()
-    old = frame.last_exception
-    frame.last_exception = err
-    if not hasattr(space, '_w_ExceptionInfo'):
-        space._w_ExceptionInfo = space.appexec([], """():
-    class _ExceptionInfo(object):
-        def __init__(self):
-            import sys
-            self.type, self.value, self.traceback = sys.exc_info()
-
-    return _ExceptionInfo
-""")
-    try:
-        return space.call_function(space._w_ExceptionInfo)
-    finally:
-        frame.last_exception = old
+def _exc_info(space, operror):
+    """sys.exc_info() isn't set until a app except block catches it,
+    but we can directly copy the two lines of code from module/sys/vm.py."""
+    operror.normalize_exception(space)
+    return space.appexec([operror.w_type, operror.get_w_value(space),
+                          space.wrap(operror.get_traceback())], """(t, v, tb):
+        class _ExceptionInfo:
+            pass
+        e = _ExceptionInfo()
+        e.type = t
+        e.value = v
+        e.traceback = tb
+        return e
+    """)
 
 def pypyraises(space, w_ExpectedException, w_expr, __args__):
     """A built-in function providing the equivalent of py.test.raises()."""

From pypy.commits at gmail.com  Wed Nov 16 11:53:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 08:53:51 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: fix a test
Message-ID: <582c8f1f.8a29c20a.26431.afbc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88409:a7f6ce45fb45
Date: 2016-11-16 16:53 +0000
http://bitbucket.org/pypy/pypy/changeset/a7f6ce45fb45/

Log:	fix a test

diff --git a/rpython/jit/backend/x86/tool/test/test_viewcode.py b/rpython/jit/backend/x86/tool/test/test_viewcode.py
--- a/rpython/jit/backend/x86/tool/test/test_viewcode.py
+++ b/rpython/jit/backend/x86/tool/test/test_viewcode.py
@@ -62,7 +62,7 @@
 def test_find_objdump():
     old = os.environ['PATH']
     os.environ['PATH'] = ''
-    py.test.raises(find_objdump)
+    py.test.raises(Exception, find_objdump)
 
     #
     path = udir.join('objdump')

From pypy.commits at gmail.com  Wed Nov 16 12:02:00 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 09:02:00 -0800 (PST)
Subject: [pypy-commit] pypy pytest-2.9.2: Close branch pytest-2.9.2
Message-ID: <582c9108.8d071c0a.d4bf.8a93@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: pytest-2.9.2
Changeset: r88410:0947dd4d79c2
Date: 2016-11-16 17:01 +0000
http://bitbucket.org/pypy/pypy/changeset/0947dd4d79c2/

Log:	Close branch pytest-2.9.2


From pypy.commits at gmail.com  Wed Nov 16 12:02:18 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 09:02:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in pytest-2.9.2 (pull request
 #496)
Message-ID: <582c911a.ca10c20a.c108e.c112@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88411:962ef070f39e
Date: 2016-11-16 17:01 +0000
http://bitbucket.org/pypy/pypy/changeset/962ef070f39e/

Log:	Merged in pytest-2.9.2 (pull request #496)

	Update pytest to 2.9.2

diff too long, truncating to 2000 out of 15596 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls

From pypy.commits at gmail.com  Wed Nov 16 12:06:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 09:06:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <582c9204.55911c0a.cd5ec.849e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88412:291df33d2844
Date: 2016-11-16 17:05 +0000
http://bitbucket.org/pypy/pypy/changeset/291df33d2844/

Log:	hg merge default

diff too long, truncating to 2000 out of 16227 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls

From pypy.commits at gmail.com  Wed Nov 16 12:16:15 2016
From: pypy.commits at gmail.com (wlav)
Date: Wed, 16 Nov 2016 09:16:15 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: first stab at migration to new
 wrapper functions
Message-ID: <582c945f.448e1c0a.143e2.b32b@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: space-newtext
Changeset: r88413:e4e75abf6313
Date: 2016-11-16 09:15 -0800
http://bitbucket.org/pypy/pypy/changeset/e4e75abf6313/

Log:	first stab at migration to new wrapper functions

diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -55,7 +55,7 @@
 
 eci = ExternalCompilationInfo(
     separate_module_files=[srcpath.join("cintcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
+    include_dirs=[incpath, translator_c_dir] + rootincpath,
     includes=["cintcwrapper.h"],
     library_dirs=rootlibpath,
     libraries=["Hist", "Core", "Cint"],
@@ -167,7 +167,7 @@
         memory_regulator.register(cppself)
 
         # tie all the life times to the TF1 instance
-        space.setattr(w_self, space.wrap('_callback'), w_callback)
+        space.setattr(w_self, space.newtext('_callback'), w_callback)
 
         # by definition for __init__
         return None
@@ -249,9 +249,9 @@
 def activate_branch(space, w_branch):
     w_branches = space.call_method(w_branch, "GetListOfBranches")
     for i in range(space.r_longlong_w(space.call_method(w_branches, "GetEntriesFast"))):
-        w_b = space.call_method(w_branches, "At", space.wrap(i))
+        w_b = space.call_method(w_branches, "At", space.newlong(i))
         activate_branch(space, w_b)
-    space.call_method(w_branch, "SetStatus", space.wrap(1))
+    space.call_method(w_branch, "SetStatus", space.newint(1))
     space.call_method(w_branch, "ResetReadEntry")
 
 c_ttree_GetEntry = rffi.llexternal(
@@ -277,7 +277,7 @@
 
     # try the saved cdata (for builtin types)
     try:
-        w_cdata = space.getattr(w_self, space.wrap('_'+attr))
+        w_cdata = space.getattr(w_self, space.newtext('_'+attr))
         from pypy.module._cffi_backend import cdataobj
         cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
         return cdata.convert_to_object()
@@ -302,15 +302,15 @@
         klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname))
         w_obj = klass.construct()
         # 0x10000 = kDeleteObject; reset because we own the object
-        space.call_method(w_branch, "ResetBit", space.wrap(0x10000))
+        space.call_method(w_branch, "ResetBit", space.newint(0x10000))
         space.call_method(w_branch, "SetObject", w_obj)
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
+        space.call_method(w_branch, "GetEntry", space.newlong(entry))
         space.setattr(w_self, args_w[0], w_obj)
         return w_obj
     else:
         # builtin data
         w_leaf = space.call_method(w_self, "GetLeaf", args_w[0])
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
+        space.call_method(w_branch, "GetEntry", space.newlong(entry))
 
         # location
         w_address = space.call_method(w_leaf, "GetValuePointer")
@@ -329,7 +329,7 @@
         cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename))
 
         # cache result
-        space.setattr(w_self, space.wrap('_'+attr), space.wrap(cdata))
+        space.setattr(w_self, space.newtext('_'+attr), space.wrap(cdata))
         return space.getattr(w_self, args_w[0])
 
 class W_TTreeIter(W_Root):
@@ -343,7 +343,7 @@
         self.maxentry = space.r_longlong_w(space.call_method(w_tree, "GetEntriesFast"))
 
         space = self.space = tree.space          # holds the class cache in State
-        space.call_method(w_tree, "SetBranchStatus", space.wrap("*"), space.wrap(0))
+        space.call_method(w_tree, "SetBranchStatus", space.newtext("*"), space.newint(0))
 
     def iter_w(self):
         return self.space.wrap(self)
@@ -397,7 +397,7 @@
         _method_alias(space, w_pycppclass, "__len__", "GetSize")
 
     elif name == "TF1":
-        space.setattr(w_pycppclass, space.wrap("__init__"), _pythonizations["tf1_tf1"])
+        space.setattr(w_pycppclass, space.newtext("__init__"), _pythonizations["tf1_tf1"])
 
     elif name == "TFile":
         _method_alias(space, w_pycppclass, "__getattr__", "Get")
@@ -415,9 +415,9 @@
     elif name == "TTree":
         _method_alias(space, w_pycppclass, "_unpythonized_Branch", "Branch")
 
-        space.setattr(w_pycppclass, space.wrap("Branch"),      _pythonizations["ttree_Branch"])
-        space.setattr(w_pycppclass, space.wrap("__iter__"),    _pythonizations["ttree_iter"])
-        space.setattr(w_pycppclass, space.wrap("__getattr__"), _pythonizations["ttree_getattr"])
+        space.setattr(w_pycppclass, space.newtext("Branch"),      _pythonizations["ttree_Branch"])
+        space.setattr(w_pycppclass, space.newtext("__iter__"),    _pythonizations["ttree_iter"])
+        space.setattr(w_pycppclass, space.newtext("__getattr__"), _pythonizations["ttree_getattr"])
 
     elif name[0:8] == "TVectorT":    # TVectorT<> template
         _method_alias(space, w_pycppclass, "__len__", "GetNoElements")
diff --git a/pypy/module/cppyy/capi/reflex_capi.py b/pypy/module/cppyy/capi/reflex_capi.py
--- a/pypy/module/cppyy/capi/reflex_capi.py
+++ b/pypy/module/cppyy/capi/reflex_capi.py
@@ -9,6 +9,10 @@
 srcpath = pkgpath.join("src")
 incpath = pkgpath.join("include")
 
+# require local translator path to pickup common defs
+from rpython.translator import cdir
+translator_c_dir = py.path.local(cdir)
+
 import commands
 (config_stat, incdir) = commands.getstatusoutput("root-config --incdir")
 
@@ -39,7 +43,7 @@
 
 eci = ExternalCompilationInfo(
     separate_module_files=[srcpath.join("reflexcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
+    include_dirs=[incpath, translator_c_dir] + rootincpath,
     includes=["reflexcwrapper.h"],
     library_dirs=rootlibpath,
     libraries=["Reflex"],
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -180,7 +180,7 @@
         self.size = sys.maxint
 
     def convert_argument(self, space, w_obj, address, call_local):
-        w_tc = space.findattr(w_obj, space.wrap('typecode'))
+        w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.str_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
                         "expected %s pointer type, but received %s",
@@ -312,7 +312,7 @@
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
-        return space.wrap(address[0])
+        return space.newbytes(address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
@@ -331,7 +331,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
-        return space.wrap(float(rffiptr[0]))
+        return space.newfloat(float(rffiptr[0]))
 
 class ConstFloatRefConverter(FloatConverter):
     _immutable_fields_ = ['libffitype', 'typecode']
@@ -370,7 +370,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
-        return space.wrap(rffi.charp2str(charpptr[0]))
+        return space.newbytes(rffi.charp2str(charpptr[0]))
 
     def free_argument(self, space, arg, call_local):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
@@ -622,7 +622,7 @@
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
-        return space.wrap(longptr[0])
+        return space.newlong(longptr[0])
 
 
 _converters = {}         # builtin and custom types
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -54,7 +54,7 @@
             raise NotImplementedError
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ptrval = rffi.cast(rffi.ULONG, lresult)
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap(self.typecode)))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext(self.typecode)))
         if ptrval == 0:
             from pypy.module.cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
@@ -128,9 +128,9 @@
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ccpresult = rffi.cast(rffi.CCHARP, lresult)
         if ccpresult == rffi.cast(rffi.CCHARP, 0):
-            return space.wrap("")
+            return space.newbytes("")
         result = rffi.charp2str(ccpresult)   # TODO: make it a choice to free
-        return space.wrap(result)
+        return space.newbytes(result)
 
 
 class ConstructorExecutor(FunctionExecutor):
@@ -139,7 +139,7 @@
         from pypy.module.cppyy import interp_cppyy
         newthis = capi.c_constructor(space, cppmethod, cpptype, num_args, args)
         assert lltype.typeOf(newthis) == capi.C_OBJECT
-        return space.wrap(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
+        return space.newlong(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
 
 
 class InstancePtrExecutor(FunctionExecutor):
@@ -196,7 +196,7 @@
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         cstr_result = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
-        return space.wrap(capi.charp2str_free(space, cstr_result))
+        return space.newbytes(capi.charp2str_free(space, cstr_result))
 
     def execute_libffi(self, space, cif_descr, funcaddr, buffer):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -26,7 +26,7 @@
         return arg
 
     def _wrap_object(self, space, obj):
-        return space.wrap(bool(ord(rffi.cast(rffi.CHAR, obj))))
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
 
 class CharTypeMixin(object):
     _mixin_     = True
@@ -153,7 +153,7 @@
         return r_singlefloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(float(obj))
+        return space.newfloat(float(obj))
 
 class DoubleTypeMixin(object):
     _mixin_     = True
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -33,7 +33,7 @@
     try:
         cdll = capi.c_load_dictionary(name)
     except rdynload.DLOpenError as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(str(e.msg)))
+        raise OperationError(space.w_RuntimeError, space.newtext(str(e.msg)))
     return W_CPPLibrary(space, cdll)
 
 class State(object):
@@ -53,7 +53,7 @@
     if state.w_nullptr is None:
         from pypy.module._rawffi.interp_rawffi import unpack_simple_shape
         from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap('P')))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext('P')))
         # TODO: fix this hack; fromaddress() will allocate memory if address
         # is null and there seems to be no way around it (ll_buffer can not
         # be touched directly)
@@ -65,7 +65,7 @@
 
 @unwrap_spec(name=str)
 def resolve_name(space, name):
-    return space.wrap(capi.c_resolve_name(space, name))
+    return space.newtext(capi.c_resolve_name(space, name))
 
 @unwrap_spec(name=str)
 def scope_byname(space, name):
@@ -113,7 +113,7 @@
     return None
 
 def std_string_name(space):
-    return space.wrap(capi.std_string_name)
+    return space.newtext(capi.std_string_name)
 
 @unwrap_spec(w_callback=W_Root)
 def set_class_generator(space, w_callback):
@@ -126,7 +126,7 @@
     state.w_fngen_callback = w_callback
 
 def register_class(space, w_pycppclass):
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
@@ -431,7 +431,7 @@
             try:
                 s = self.space.str_w(args_w[i])
             except OperationError:
-                s = self.space.str_w(self.space.getattr(args_w[i], self.space.wrap('__name__')))
+                s = self.space.str_w(self.space.getattr(args_w[i], self.space.newtext('__name__')))
             s = capi.c_resolve_name(self.space, s)
             if s != self.templ_args[i]:
                 raise oefmt(self.space.w_TypeError,
@@ -537,7 +537,7 @@
         # only get here if all overloads failed ...
         errmsg = 'none of the %d overloaded methods succeeded. Full details:' % len(self.functions)
         if hasattr(self.space, "fake"):     # FakeSpace fails errorstr (see below)
-            raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+            raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
@@ -554,13 +554,13 @@
                 errmsg += '\n  '+cppyyfunc.signature()+' =>\n'
                 errmsg += '    Exception: '+str(e)
 
-        raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+        raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
 
     def signature(self):
         sig = self.functions[0].signature()
         for i in range(1, len(self.functions)):
             sig += '\n'+self.functions[i].signature()
-        return self.space.wrap(sig)
+        return self.space.newtext(sig)
 
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.signature() for f in self.functions]
@@ -633,7 +633,7 @@
         self.offset = offset
 
     def get_returntype(self):
-        return self.space.wrap(self.converter.name)
+        return self.space.newtext(self.converter.name)
 
     def _get_offset(self, cppinstance):
         if cppinstance:
@@ -750,7 +750,7 @@
         return capi.c_scoped_final_name(self.space, self.handle)
 
     def get_method_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.methods])
+        return self.space.newlist([self.space.newtext(name) for name in self.methods])
 
     def get_overload(self, name):
         try:
@@ -762,7 +762,7 @@
         return new_method
 
     def get_datamember_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.datamembers])
+        return self.space.newlist([self.space.newtext(name) for name in self.datamembers])
 
     def get_datamember(self, name):
         try:
@@ -856,17 +856,17 @@
         alldir = []
         for i in range(capi.c_num_scopes(self.space, self)):
             sname = capi.c_scope_name(self.space, self, i)
-            if sname: alldir.append(self.space.wrap(sname))
+            if sname: alldir.append(self.space.newtext(sname))
         allmeth = {}
         for i in range(capi.c_num_methods(self.space, self)):
             idx = capi.c_method_index_at(self.space, self, i)
             mname = capi.c_method_name(self.space, self, idx)
             if mname: allmeth.setdefault(mname, 0)
         for m in allmeth.keys():
-            alldir.append(self.space.wrap(m))
+            alldir.append(self.space.newtext(m))
         for i in range(capi.c_num_datamembers(self.space, self)):
             dname = capi.c_datamember_name(self.space, self, i)
-            if dname: alldir.append(self.space.wrap(dname))
+            if dname: alldir.append(self.space.newtext(dname))
         return self.space.newlist(alldir)
         
 
@@ -952,7 +952,7 @@
         num_bases = capi.c_num_bases(self.space, self)
         for i in range(num_bases):
             base_name = capi.c_base_name(self.space, self, i)
-            bases.append(self.space.wrap(base_name))
+            bases.append(self.space.newtext(base_name))
         return self.space.newlist(bases)
 
 W_CPPClass.typedef = TypeDef(
@@ -1038,7 +1038,8 @@
         self._opt_register_finalizer()
 
     def _opt_register_finalizer(self):
-        if self.python_owns and not self.finalizer_registered:
+        if self.python_owns and not self.finalizer_registered \
+               and not hasattr(self.space, "fake"):
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 
@@ -1049,7 +1050,7 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.wrap(self.python_owns)
+        return space.newbool(self.python_owns)
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
@@ -1091,7 +1092,7 @@
     def instance__eq__(self, w_other):
         # special case: if other is None, compare pointer-style
         if self.space.is_w(w_other, self.space.w_None):
-            return self.space.wrap(not self._rawobject)
+            return self.space.newbool(not self._rawobject)
 
         # get here if no class-specific overloaded operator is available, try to
         # find a global overload in gbl, in __gnu_cxx (for iterators), or in the
@@ -1121,7 +1122,7 @@
         # the first data member in a struct and the struct have the same address)
         other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)  # TODO: factor out
         iseq = (self._rawobject == other._rawobject) and (self.cppclass == other.cppclass)
-        return self.space.wrap(iseq)
+        return self.space.newbool(iseq)
 
     def instance__ne__(self, w_other):
         return self.space.not_(self.instance__eq__(w_other))
@@ -1149,7 +1150,7 @@
         w_as_builtin = self._get_as_builtin()
         if w_as_builtin is not None:
             return self.space.repr(w_as_builtin)
-        return self.space.wrap("<%s object at 0x%x>" %
+        return self.space.newtext("<%s object at 0x%x>" %
                                (self.cppclass.name, rffi.cast(rffi.ULONG, self.get_rawobject())))
 
     def destruct(self):
@@ -1215,12 +1216,12 @@
     except KeyError:
         final_name = capi.c_scoped_final_name(space, handle)
         # the callback will cache the class by calling register_class
-        w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name))
+        w_pycppclass = space.call_function(state.w_clgen_callback, space.newtext(final_name))
     return w_pycppclass
 
 def get_interface_func(space, w_callable, npar):
     state = space.fromcache(State)
-    return space.call_function(state.w_fngen_callback, w_callable, space.wrap(npar))
+    return space.call_function(state.w_fngen_callback, w_callable, space.newint(npar))
 
 def wrap_cppobject(space, rawobject, cppclass,
                    do_cast=True, python_owns=False, is_ref=False, fresh=False):
@@ -1235,7 +1236,7 @@
                 w_pycppclass = get_pythonized_cppclass(space, actual)
                 offset = capi.c_base_offset1(space, actual, cppclass, rawobject, -1)
                 rawobject = capi.direct_ptradd(rawobject, offset)
-                w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+                w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
                 cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
             except Exception:
                 # failed to locate/build the derived class, so stick to the base (note
@@ -1272,7 +1273,7 @@
 def addressof(space, w_obj):
     """Takes a bound C++ instance or array, returns the raw address."""
     address = _addressof(space, w_obj)
-    return space.wrap(address)
+    return space.newlong(address)
 
 @unwrap_spec(owns=bool, cast=bool)
 def bind_object(space, w_obj, w_pycppclass, owns=False, cast=False):
@@ -1283,7 +1284,7 @@
     except Exception:
         # accept integer value as address
         rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj))
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     if not w_cppclass:
         w_cppclass = scope_byname(space, space.str_w(w_pycppclass))
         if not w_cppclass:
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -18,8 +18,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct = cls.space.wrap(test_dct)
-        cls.w_capi_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct = cls.space.newtext(test_dct)
+        cls.w_capi_identity = cls.space.newtext(capi.identify())
         cls.w_advanced = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py
--- a/pypy/module/cppyy/test/test_cint.py
+++ b/pypy/module/cppyy/test/test_cint.py
@@ -169,11 +169,11 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)
-        cls.w_M = cls.space.wrap(10)
-        cls.w_fname = cls.space.wrap("test.root")
-        cls.w_tname = cls.space.wrap("test")
-        cls.w_title = cls.space.wrap("test tree")
+        cls.w_N = cls.space.newint(5)
+        cls.w_M = cls.space.newint(10)
+        cls.w_fname = cls.space.newtext("test.root")
+        cls.w_tname = cls.space.newtext("test")
+        cls.w_title = cls.space.newtext("test tree")
         cls.w_iotypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (iotypes_dct,))
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -9,8 +9,6 @@
 from pypy.module.cpyext import api
 from pypy.module.cpyext.state import State
 
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("crossingDict.so"))
@@ -24,7 +22,7 @@
 
 # from pypy/module/cpyext/test/test_cpyext.py; modified to accept more external
 # symbols and called directly instead of import_module
-def compile_extension_module(space, modname, symbols, **kwds):
+def compile_extension_module(space, modname, **kwds):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -38,19 +36,23 @@
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        kwds["libraries"] = []#[api_library]
         # '%s' undefined; assuming extern returning int
         kwds["compile_extra"] = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+                              (space.int_w(w_maj), space.int_w(w_min))]
     elif sys.platform == 'darwin':
         kwds["link_files"] = [str(api_library + '.dylib')]
     else:
         kwds["link_files"] = [str(api_library + '.so')]
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror=implicit-function-declaration"]
+            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
+            kwds["link_extra"]=["-g"]
 
     modname = modname.split('.')[-1]
     eci = ExternalCompilationInfo(
-        #export_symbols=['init%s' % (modname,)]+symbols,
         include_dirs=api.include_dirs,
         **kwds
         )
@@ -65,28 +67,30 @@
     soname.rename(pydname)
     return str(pydname)
 
-class AppTestCrossing(AppTestCpythonExtensionBase):
-    spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools', 'cpyext'])
+class AppTestCrossing:
+    spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        AppTestCpythonExtensionBase.setup_class.im_func(cls)
         # cppyy specific additions (note that test_dct is loaded late
         # to allow the generated extension module be loaded first)
-        cls.w_test_dct    = cls.space.wrap(test_dct)
+        cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
-            import cppyy, cpyext, ctypes""")    # prevents leak-checking complaints on ctypes
+            import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
-        AppTestCpythonExtensionBase.setup_method.im_func(self, func)
-
         @unwrap_spec(name=str, init=str, body=str)
-        def create_cdll(space, name, init, body, w_symbols):
+        def create_cdll(space, name, init, body):
             # the following is loosely from test_cpyext.py import_module; it
             # is copied here to be able to tweak the call to
             # compile_extension_module and to get a different return result
             # than in that function
             code = """
             #include <Python.h>
+            /* fix for cpython 2.7 Python.h if running tests with -A
+               since pypy compiles with -fvisibility-hidden */
+            #undef PyMODINIT_FUNC
+            #define PyMODINIT_FUNC RPY_EXPORTED void
+
             %(body)s
 
             PyMODINIT_FUNC
@@ -95,28 +99,16 @@
             }
             """ % dict(name=name, init=init, body=body)
             kwds = dict(separate_module_sources=[code])
-            symbols = [space.str_w(w_item) for w_item in space.fixedview(w_symbols)]
-            mod = compile_extension_module(space, name, symbols, **kwds)
+            mod = compile_extension_module(space, name, **kwds)
 
             # explicitly load the module as a CDLL rather than as a module
             from pypy.module.imp.importing import get_so_extension
             fullmodname = os.path.join(
                 os.path.dirname(mod), name + get_so_extension(space))
-            return space.wrap(fullmodname)
+            return space.newtext(fullmodname)
 
         self.w_create_cdll = self.space.wrap(interp2app(create_cdll))
 
-    def test00_base_class(self):
-        """Test from cpyext; only here to see whether the imported class works"""
-
-        import sys
-        init = """
-        if (Py_IsInitialized())
-            Py_InitModule("foo", NULL);
-        """
-        self.import_module(name='foo', init=init)
-        assert 'foo' in sys.modules
-
     def test01_build_bar_extension(self):
         """Test that builds the needed extension; runs as test to keep it loaded"""
 
@@ -131,10 +123,12 @@
 
         # note: only the symbols are needed for C, none for python
         body = """
+        RPY_EXPORTED
         long bar_unwrap(PyObject* arg)
         {
-            return PyLong_AsLong(arg);
+            return 13;//PyLong_AsLong(arg);
         }
+        RPY_EXPORTED
         PyObject* bar_wrap(long l)
         {
             return PyLong_FromLong(l);
@@ -146,8 +140,7 @@
         # explicitly load the module as a CDLL rather than as a module
         import ctypes
         self.cmodule = ctypes.CDLL(
-            self.create_cdll(name, init, body, ['bar_unwrap', 'bar_wrap']),
-            ctypes.RTLD_GLOBAL)
+            self.create_cdll(name, init, body), ctypes.RTLD_GLOBAL)
 
     def test02_crossing_dict(self):
         """Test availability of all needed classes in the dict"""
@@ -160,6 +153,7 @@
 
         assert crossing.A == crossing.A
 
+    @py.test.mark.dont_track_allocations("fine when running standalone, though?!")
     def test03_send_pyobject(self):
         """Test sending a true pyobject to C++"""
 
@@ -169,6 +163,7 @@
         a = crossing.A()
         assert a.unwrap(13) == 13
 
+    @py.test.mark.dont_track_allocations("fine when running standalone, though?!")
     def test04_send_and_receive_pyobject(self):
         """Test receiving a true pyobject from C++"""
 
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)  # should be imported from the dictionary
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -17,8 +17,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
-        cls.w_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_identity = cls.space.newtext(capi.identify())
         cls.w_fragile = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_operators.py b/pypy/module/cppyy/test/test_operators.py
--- a/pypy/module/cppyy/test/test_operators.py
+++ b/pypy/module/cppyy/test/test_operators.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)  # should be imported from the dictionary
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_overloads.py b/pypy/module/cppyy/test/test_overloads.py
--- a/pypy/module/cppyy/test/test_overloads.py
+++ b/pypy/module/cppyy/test/test_overloads.py
@@ -16,7 +16,7 @@
 
     def setup_class(cls):
         env = os.environ
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -17,7 +17,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -376,7 +376,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -203,7 +203,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -279,8 +279,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -335,8 +335,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -444,7 +444,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -481,7 +481,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_streams.py b/pypy/module/cppyy/test/test_streams.py
--- a/pypy/module/cppyy/test/test_streams.py
+++ b/pypy/module/cppyy/test/test_streams.py
@@ -15,7 +15,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_streams = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -3,14 +3,17 @@
 from rpython.rlib.objectmodel import specialize, instantiate
 from rpython.rlib import rarithmetic, jit
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper import llinterp
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
 
-from pypy.module.cppyy import interp_cppyy, capi
+from pypy.module.cppyy import interp_cppyy, capi, executor
 # These tests are for the backend that support the fast path only.
 if capi.identify() == 'CINT':
     py.test.skip("CINT does not support fast path")
 elif capi.identify() == 'loadable_capi':
     py.test.skip("can not currently use FakeSpace with _cffi_backend")
+elif os.getenv("CPPYY_DISABLE_FASTPATH"):
+    py.test.skip("fast path is disabled by CPPYY_DISABLE_FASTPATH envar")
 
 # load cpyext early, or its global vars are counted as leaks in the test
 # (note that the module is not otherwise used in the test itself)
@@ -29,6 +32,23 @@
     return rffi.ptradd(ptr, offset)
 capi.exchange_address = _opaque_exchange_address
 
+# add missing alt_errno (??)
+def get_tlobj(self):
+    try:
+        return self._tlobj
+    except AttributeError:
+        from rpython.rtyper.lltypesystem import rffi
+        PERRNO = rffi.CArrayPtr(rffi.INT)
+        fake_p_errno = lltype.malloc(PERRNO.TO, 1, flavor='raw', zero=True,
+                                     track_allocation=False)
+        self._tlobj = {'RPY_TLOFS_p_errno': fake_p_errno,
+                       'RPY_TLOFS_alt_errno': rffi.cast(rffi.INT, 0),
+                       #'thread_ident': ...,
+                       }
+        return self._tlobj
+llinterp.LLInterpreter.get_tlobj = get_tlobj
+
+
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
@@ -43,6 +63,10 @@
 class FakeBase(W_Root):
     typename = None
 
+class FakeBool(FakeBase):
+    typename = "bool"
+    def __init__(self, val):
+        self.val = val
 class FakeInt(FakeBase):
     typename = "int"
     def __init__(self, val):
@@ -69,9 +93,10 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 class FakeException(FakeType):
-    def __init__(self, name):
+    def __init__(self, space, name):
         FakeType.__init__(self, name)
-        self.message = name
+        self.msg = name
+        self.space = space
 
 class FakeUserDelAction(object):
     def __init__(self, space):
@@ -83,16 +108,13 @@
     def perform(self, executioncontext, frame):
         pass
 
+class FakeState(object):
+    def __init__(self, space):
+        self.slowcalls = 0
+
 class FakeSpace(object):
     fake = True
 
-    w_ValueError = FakeException("ValueError")
-    w_TypeError = FakeException("TypeError")
-    w_AttributeError = FakeException("AttributeError")
-    w_ReferenceError = FakeException("ReferenceError")
-    w_NotImplementedError = FakeException("NotImplementedError")
-    w_RuntimeError = FakeException("RuntimeError")
-
     w_None = None
     w_str = FakeType("str")
     w_int = FakeType("int")
@@ -105,6 +127,21 @@
         self.config = dummy()
         self.config.translating = False
 
+        # kill calls to c_call_i (i.e. slow path)
+        def c_call_i(space, cppmethod, cppobject, nargs, args):
+            assert not "slow path called"
+            return capi.c_call_i(space, cppmethod, cppobject, nargs, args)
+        executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i)
+
+        self.w_AttributeError      = FakeException(self, "AttributeError")
+        self.w_KeyError            = FakeException(self, "KeyError")
+        self.w_NotImplementedError = FakeException(self, "NotImplementedError")
+        self.w_ReferenceError      = FakeException(self, "ReferenceError")
+        self.w_RuntimeError        = FakeException(self, "RuntimeError")
+        self.w_SystemError         = FakeException(self, "SystemError")
+        self.w_TypeError           = FakeException(self, "TypeError")
+        self.w_ValueError          = FakeException(self, "ValueError")
+
     def issequence_w(self, w_obj):
         return True
 
@@ -120,6 +157,30 @@
             return FakeInt(int(obj))
         assert 0
 
+    @specialize.argtype(1)
+    def newbool(self, obj):
+        return FakeBool(obj)
+
+    @specialize.argtype(1)
+    def newint(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newlong(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newfloat(self, obj):
+        return FakeFloat(obj)
+
+    @specialize.argtype(1)
+    def newbytes(self, obj):
+        return FakeString(obj)
+
+    @specialize.argtype(1)
+    def newtext(self, obj):
+        return FakeString(obj)
+
     def float_w(self, w_obj, allow_conversion=True):
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
@@ -210,7 +271,8 @@
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(1)
+        self.check_jitcell_token_count(1)   # same for fast and slow path??
+        # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     def test01_simple(self):
         """Test fast path being taken for methods"""

From pypy.commits at gmail.com  Wed Nov 16 12:26:11 2016
From: pypy.commits at gmail.com (sbauman)
Date: Wed, 16 Nov 2016 09:26:11 -0800 (PST)
Subject: [pypy-commit] pypy clean-exported-state: Merge with default
Message-ID: <582c96b3.45ad1c0a.69eeb.bdac@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: clean-exported-state
Changeset: r88414:9f494a105d18
Date: 2016-11-16 12:25 -0500
http://bitbucket.org/pypy/pypy/changeset/9f494a105d18/

Log:	Merge with default

diff too long, truncating to 2000 out of 31250 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:

From pypy.commits at gmail.com  Wed Nov 16 13:35:43 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 Nov 2016 10:35:43 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: (half cfbolz half
 arigo) kill some more dead code, and fix fset_f_lineno
Message-ID: <582ca6ff.c5371c0a.a5349.cd36@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88415:d45402cf7644
Date: 2016-11-16 18:38 +0000
http://bitbucket.org/pypy/pypy/changeset/d45402cf7644/

Log:	(half cfbolz half arigo) kill some more dead code, and fix
	fset_f_lineno

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -46,44 +46,6 @@
                            self.get_qualname(),
                            unicode(addrstring)))
 
-    def descr__reduce__(self, space):
-        # DEAD CODE, see frame.__reduce__
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod = space.getbuiltinmodule('_pickle_support')
-        mod = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get(self.KIND + '_new')
-        w = space.wrap
-        if self.frame:
-            w_frame = self.frame._reduce_state(space)
-        else:
-            w_frame = space.w_None
-
-        tup = [w_frame, w(self.running)]
-        return space.newtuple([new_inst, space.newtuple([]),
-                               space.newtuple(tup)])
-
-    def descr__setstate__(self, space, w_args):
-        # DEAD CODE, see frame.__reduce__
-        from rpython.rlib.objectmodel import instantiate
-        args_w = space.unpackiterable(w_args)
-        w_framestate, w_running = args_w
-        if space.is_w(w_framestate, space.w_None):
-            self.frame = None
-            self.space = space
-            self.pycode = None
-            self._name = None
-            self._qualname = None
-        else:
-            frame = instantiate(space.FrameClass)   # XXX fish
-            frame.descr__setstate__(space, w_framestate)
-            if isinstance(self, GeneratorIterator):
-                GeneratorIterator.__init__(self, frame)
-            elif isinstance(self, Coroutine):
-                Coroutine.__init__(self, frame)
-            else:
-                assert False
-        self.running = self.space.is_true(w_running)
-
     def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator/coroutine,
 return next yielded value or raise StopIteration."""
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -21,7 +21,7 @@
 
 # Define some opcodes used
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY SETUP_WITH
-POP_BLOCK END_FINALLY'''.split():
+SETUP_ASYNC_WITH POP_BLOCK END_FINALLY'''.split():
     globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
@@ -455,136 +455,6 @@
                 self.space, arguments, keywords, keywords_w, w_star,
                 w_starstar, methodcall=methodcall)
 
-    @jit.dont_look_inside
-    def descr__reduce__(self, space):
-        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
-        # Either re-add at some point, or kill this code.
-        dead_code
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod    = space.getbuiltinmodule('_pickle_support')
-        mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('frame_new')
-        w_tup_state = self._reduce_state(space)
-        nt = space.newtuple
-        return nt([new_inst, nt([]), w_tup_state])
-
-    @jit.dont_look_inside
-    def _reduce_state(self, space):
-        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
-        dead_code
-        from pypy.module._pickle_support import maker # helper fns
-        w = space.wrap
-        nt = space.newtuple
-
-        if self.get_w_f_trace() is None:
-            f_lineno = self.get_last_lineno()
-        else:
-            f_lineno = self.getorcreatedebug().f_lineno
-
-        nlocals = self.pycode.co_nlocals
-        values_w = self.locals_cells_stack_w
-        w_locals_cells_stack = maker.slp_into_tuple_with_nulls(space, values_w)
-
-        w_blockstack = nt([block._get_state_(space) for block in self.get_blocklist()])
-        if self.last_exception is None:
-            w_exc_value = space.w_None
-            w_tb = space.w_None
-        else:
-            w_exc_value = self.last_exception.get_w_value(space)
-            w_tb = w(self.last_exception.get_traceback())
-
-        d = self.getorcreatedebug()
-        tup_state = [
-            w(self.f_backref()),
-            w(self.get_builtin()),
-            w(self.pycode),
-            w_locals_cells_stack,
-            w_blockstack,
-            w_exc_value, # last_exception
-            w_tb,        #
-            self.get_w_globals(),
-            w(self.last_instr),
-            w(self.frame_finished_execution),
-            w(f_lineno),
-            space.w_None,           #XXX placeholder for f_locals
-
-            #f_restricted requires no additional data!
-            space.w_None,
-
-            w(d.instr_lb),
-            w(d.instr_ub),
-            w(d.instr_prev_plus_one),
-            w(self.valuestackdepth),
-            ]
-        return nt(tup_state)
-
-    @jit.dont_look_inside
-    def descr__setstate__(self, space, w_args):
-        # DEAD CODE AHEAD: frame.__reduce__() has been removed.
-        # Either re-add at some point, or kill this code.
-        dead_code
-        from pypy.module._pickle_support import maker # helper fns
-        from pypy.interpreter.pycode import PyCode
-        from pypy.interpreter.module import Module
-        args_w = space.unpackiterable(w_args, 17)
-        w_f_back, w_builtin, w_pycode, w_locals_cells_stack, w_blockstack, w_exc_value, w_tb,\
-            w_globals, w_last_instr, w_finished, w_f_lineno, w_f_locals, \
-            w_f_trace, w_instr_lb, w_instr_ub, w_instr_prev_plus_one, w_stackdepth = args_w
-
-        new_frame = self
-        pycode = space.interp_w(PyCode, w_pycode)
-
-        values_w = maker.slp_from_tuple_with_nulls(space, w_locals_cells_stack)
-        nfreevars = len(pycode.co_freevars)
-        closure = None
-        if nfreevars:
-            base = pycode.co_nlocals + len(pycode.co_cellvars)
-            closure = values_w[base: base + nfreevars]
-
-        # do not use the instance's __init__ but the base's, because we set
-        # everything like cells from here
-        # XXX hack
-        from pypy.interpreter.function import Function
-        outer_func = Function(space, None, closure=closure,
-                             forcename="fake")
-        PyFrame.__init__(self, space, pycode, w_globals, outer_func)
-        f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True)
-        new_frame.f_backref = jit.non_virtual_ref(f_back)
-
-        if space.config.objspace.honor__builtins__:
-            new_frame.builtin = space.interp_w(Module, w_builtin)
-        else:
-            assert space.interp_w(Module, w_builtin) is space.builtin
-        new_frame.set_blocklist([unpickle_block(space, w_blk)
-                                 for w_blk in space.unpackiterable(w_blockstack)])
-        self.locals_cells_stack_w = values_w[:]
-        valuestackdepth = space.int_w(w_stackdepth)
-        if not self._check_stack_index(valuestackdepth):
-            raise oefmt(space.w_ValueError, "invalid stackdepth")
-        assert valuestackdepth >= 0
-        self.valuestackdepth = valuestackdepth
-        if space.is_w(w_exc_value, space.w_None):
-            new_frame.last_exception = None
-        else:
-            from pypy.interpreter.pytraceback import PyTraceback
-            tb = space.interp_w(PyTraceback, w_tb)
-            new_frame.last_exception = OperationError(space.type(w_exc_value),
-                                                      w_exc_value, tb
-                                                      )
-        new_frame.last_instr = space.int_w(w_last_instr)
-        new_frame.frame_finished_execution = space.is_true(w_finished)
-        d = new_frame.getorcreatedebug()
-        d.f_lineno = space.int_w(w_f_lineno)
-
-        if space.is_w(w_f_trace, space.w_None):
-            d.w_f_trace = None
-        else:
-            d.w_f_trace = w_f_trace
-
-        d.instr_lb = space.int_w(w_instr_lb)   #the three for tracing
-        d.instr_ub = space.int_w(w_instr_ub)
-        d.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one)
-
     def hide(self):
         return self.pycode.hidden_applevel
 
@@ -729,7 +599,7 @@
             return space.wrap(self.getorcreatedebug().f_lineno)
 
     def fset_f_lineno(self, space, w_new_lineno):
-        "Returns the line number of the instruction currently being executed."
+        "Change the line number of the instruction currently being executed."
         try:
             new_lineno = space.int_w(w_new_lineno)
         except OperationError:
@@ -768,35 +638,41 @@
                         "can't jump to 'except' line as there's no exception")
 
         # Don't jump into or out of a finally block.
-        f_lasti_setup_addr = -1
-        new_lasti_setup_addr = -1
-        blockstack = []
+        # Unlike CPython, we can't jump into or out of an except block
+        # either---there would be a mess with SysExcInfoRestorer.
+        f_lasti_handler_addr = -1
+        new_lasti_handler_addr = -1
+        blockstack = [-1]    # list of odd length:
+                             #   * addr of most recent outermost handler
+                             # [ * addr of start of outermost block
+                             #   * addr of most recent handler in that block 
+                             #       (last two items repeated) ]
         addr = 0
         while addr < len(code):
             op = ord(code[addr])
-            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
-                blockstack.append([addr, False])
+            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
+                      SETUP_ASYNC_WITH):
+                blockstack.append(addr)
+                blockstack.append(-1)
             elif op == POP_BLOCK:
-                setup_op = ord(code[blockstack[-1][0]])
-                if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
-                    blockstack[-1][1] = True
-                else:
-                    blockstack.pop()
-            elif op == END_FINALLY:
-                if len(blockstack) > 0:
-                    setup_op = ord(code[blockstack[-1][0]])
-                    if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
-                        blockstack.pop()
+                assert len(blockstack) >= 3
+                blockstack.pop()
+                setup_op = ord(code[blockstack.pop()])
+                if setup_op != SETUP_LOOP:
+                    blockstack[-1] = addr
+            elif op == END_FINALLY:    # "async for" nests blocks
+                blockstack[-1] = -1    # strangely, careful here
 
             if addr == new_lasti or addr == self.last_instr:
-                for ii in range(len(blockstack)):
-                    setup_addr, in_finally = blockstack[~ii]
-                    if in_finally:
-                        if addr == new_lasti:
-                            new_lasti_setup_addr = setup_addr
-                        if addr == self.last_instr:
-                            f_lasti_setup_addr = setup_addr
-                        break
+                ii = len(blockstack) - 1
+                while ii > 0 and blockstack[ii] == -1:
+                    ii -= 2
+                assert ii >= 0
+                handler_addr = blockstack[ii]
+                if addr == new_lasti:
+                    new_lasti_setup_addr = handler_addr
+                if addr == self.last_instr:
+                    f_lasti_setup_addr = handler_addr
 
             if op >= HAVE_ARGUMENT:
                 addr += 3
@@ -805,11 +681,15 @@
 
         assert len(blockstack) == 0
 
-        if new_lasti_setup_addr != f_lasti_setup_addr:
+        if new_lasti_handler_addr != f_lasti_handler_addr:
             raise oefmt(space.w_ValueError,
-                        "can't jump into or out of a 'finally' block %d -> %d",
-                        f_lasti_setup_addr, new_lasti_setup_addr)
+                        "can't jump into or out of an 'expect' or "
+                        "'finally' block (%d -> %d)",
+                        f_lasti_handler_addr, new_lasti_setup_addr)
 
+        # now we know we're not jumping into or out of a place which
+        # needs a SysExcInfoRestorer.  Check that we're not jumping
+        # *into* a block, but only (potentially) out of some blocks.
         if new_lasti < self.last_instr:
             min_addr = new_lasti
             max_addr = self.last_instr
@@ -817,42 +697,40 @@
             min_addr = self.last_instr
             max_addr = new_lasti
 
-        delta_iblock = min_delta_iblock = 0
+        delta_iblock = min_delta_iblock = 0    # see below for comment
         addr = min_addr
         while addr < max_addr:
             op = ord(code[addr])
 
-            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
+            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
+                      SETUP_ASYNC_WITH):
                 delta_iblock += 1
             elif op == POP_BLOCK:
                 delta_iblock -= 1
                 if delta_iblock < min_delta_iblock:
                     min_delta_iblock = delta_iblock
 
-            if op >= stdlib_opcode.HAVE_ARGUMENT:
+            if op >= HAVE_ARGUMENT:
                 addr += 3
             else:
                 addr += 1
 
-        f_iblock = 0
-        block = self.lastblock
-        while block:
-            f_iblock += 1
-            block = block.previous
-        min_iblock = f_iblock + min_delta_iblock
+        # 'min_delta_iblock' is <= 0; its absolute value is the number of
+        # blocks we exit.  'go_iblock' is the delta number of blocks
+        # between the last_instr and the new_lasti, in this order.
         if new_lasti > self.last_instr:
-            new_iblock = f_iblock + delta_iblock
+            go_iblock = delta_iblock
         else:
-            new_iblock = f_iblock - delta_iblock
+            go_iblock = -delta_iblock
 
-        if new_iblock > min_iblock:
+        if go_iblock > min_delta_iblock:
             raise oefmt(space.w_ValueError,
                         "can't jump into the middle of a block")
+        assert go_iblock <= 0
 
-        while f_iblock > new_iblock:
+        for ii in range(-go_iblock):
             block = self.pop_block()
-            block.cleanup(self)
-            f_iblock -= 1
+            block.cleanupstack(self)
 
         self.getorcreatedebug().f_lineno = new_lineno
         self.last_instr = new_lasti
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -810,8 +810,8 @@
 
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
-    __reduce__   = interp2app(Coroutine.descr__reduce__),
-    __setstate__ = interp2app(Coroutine.descr__setstate__),
+    #__reduce__   = interp2app(Coroutine.descr__reduce__),
+    #__setstate__ = interp2app(Coroutine.descr__setstate__),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
diff --git a/pypy/module/_pickle_support/__init__.py b/pypy/module/_pickle_support/__init__.py
--- a/pypy/module/_pickle_support/__init__.py
+++ b/pypy/module/_pickle_support/__init__.py
@@ -22,5 +22,4 @@
         'intrangeiter_new': 'maker.intrangeiter_new',
         'builtin_code': 'maker.builtin_code',
         'builtin_function' : 'maker.builtin_function',
-        'operationerror_new': 'maker.operationerror_new',
     }
diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -69,10 +69,6 @@
     new_iter = W_IntRangeIterator(space, current, remaining, step)
     return space.wrap(new_iter)
 
-def operationerror_new(space):
-    from pypy.interpreter.pyopcode import W_OperationError
-    return W_OperationError(None)
-
 @unwrap_spec(identifier=str)
 def builtin_code(space, identifier):
     from pypy.interpreter import gateway

From pypy.commits at gmail.com  Wed Nov 16 13:40:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 16 Nov 2016 10:40:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: another test,
 written yesterday, which passes now on this branch
Message-ID: <582ca82a.44e61c0a.ad943.d4d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88416:1ad303c4a87c
Date: 2016-11-16 18:50 +0000
http://bitbucket.org/pypy/pypy/changeset/1ad303c4a87c/

Log:	another test, written yesterday, which passes now on this branch

diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -79,6 +79,25 @@
             assert sys.exc_info()[0] is ValueError
         assert sys.exc_info() == (None, None, None)
 
+    def test_revert_exc_info_2_finally(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        try:
+            try:
+                raise ValueError
+            finally:
+                try:
+                    try:
+                        raise IndexError
+                    finally:
+                        assert sys.exc_info()[0] is IndexError
+                except IndexError:
+                    pass
+                assert sys.exc_info()[0] is ValueError
+        except ValueError:
+            pass
+        assert sys.exc_info() == (None, None, None)
+
     def test_reraise_1(self):
         raises(IndexError, """
             import sys

From pypy.commits at gmail.com  Wed Nov 16 16:32:23 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 13:32:23 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: extend the hack from
 48bcbfaa8980 for tp_dealloc
Message-ID: <582cd067.572d1c0a.e8682.0d29@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88417:4e2740f2ebdd
Date: 2016-11-16 22:27 +0200
http://bitbucket.org/pypy/pypy/changeset/4e2740f2ebdd/

Log:	extend the hack from 48bcbfaa8980 for tp_dealloc

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -503,9 +503,13 @@
     base = pto
     this_func_ptr = llhelper(subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
-    while base.c_tp_dealloc == this_func_ptr:
+    w_obj = from_ref(space, rffi.cast(PyObject, base))
+    # see comment in userslot.slot_tp_new, this call can also infinitely recurse if
+    # called with a c-extension type that inherits from a non-c-extension type
+    while base.c_tp_dealloc == this_func_ptr or w_obj.is_cpytype():
         base = base.c_tp_base
         assert base
+        w_obj = from_ref(space, rffi.cast(PyObject, base))
     dealloc = base.c_tp_dealloc
     # XXX call tp_del if necessary
     generic_cpy_call(space, dealloc, obj)

From pypy.commits at gmail.com  Wed Nov 16 16:32:25 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 13:32:25 -0800 (PST)
Subject: [pypy-commit] pypy default: do not use -Wimplicit by default since
 it does not exist for c++ compiler
Message-ID: <582cd069.c89cc20a.57d54.a8e2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88418:b7b97f5e6bba
Date: 2016-11-16 22:28 +0200
http://bitbucket.org/pypy/pypy/changeset/b7b97f5e6bba/

Log:	do not use -Wimplicit by default since it does not exist for c++
	compiler

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -127,7 +127,9 @@
         setattr(compiler, executable, command)
 
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
+        # compiler_so can be c++ which has no -Wimplicit
+        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
+        compiler.compiler_so.extend(['-O2', '-fPIC'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])

From pypy.commits at gmail.com  Wed Nov 16 16:32:27 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 Nov 2016 13:32:27 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <582cd06b.4673c20a.25d8.26f5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88419:d6286496a013
Date: 2016-11-16 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d6286496a013/

Log:	merge default into branch

diff too long, truncating to 2000 out of 15610 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls

From pypy.commits at gmail.com  Wed Nov 16 20:33:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 17:33:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Also test directories inside
 lib-python/3/test, not just plain files
Message-ID: <582d08df.2aa9c20a.88c0a.249a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88420:e6b9354a3e71
Date: 2016-11-17 01:32 +0000
http://bitbucket.org/pypy/pypy/changeset/e6b9354a3e71/

Log:	Also test directories inside lib-python/3/test, not just plain files

diff --git a/lib-python/3/test/test_importlib/__main__.py b/lib-python/3/test/test_importlib/__main__.py
--- a/lib-python/3/test/test_importlib/__main__.py
+++ b/lib-python/3/test/test_importlib/__main__.py
@@ -1,4 +1,5 @@
 from . import load_tests
+print('loaded tests')
 import unittest
 
 unittest.main()
diff --git a/lib-python/3/unittest/main.py b/lib-python/3/unittest/main.py
--- a/lib-python/3/unittest/main.py
+++ b/lib-python/3/unittest/main.py
@@ -59,6 +59,7 @@
                     testRunner=None, testLoader=loader.defaultTestLoader,
                     exit=True, verbosity=1, failfast=None, catchbreak=None,
                     buffer=None, warnings=None, *, tb_locals=False):
+        print('TestProgram()')
         if isinstance(module, str):
             self.module = __import__(module)
             for part in module.split('.')[1:]:
@@ -74,6 +75,7 @@
         self.verbosity = verbosity
         self.buffer = buffer
         self.tb_locals = tb_locals
+        print('1')
         if warnings is None and not sys.warnoptions:
             # even if DeprecationWarnings are ignored by default
             # print them anyway unless other warnings settings are
@@ -90,7 +92,9 @@
         self.testRunner = testRunner
         self.testLoader = testLoader
         self.progName = os.path.basename(argv[0])
+        print('parsing')
         self.parseArgs(argv)
+        print('running')
         self.runTests()
 
     def usageExit(self, msg=None):
@@ -111,6 +115,7 @@
             print(MODULE_EXAMPLES % {'prog': self.progName})
 
     def parseArgs(self, argv):
+        print('in parseArgs()')
         self._initArgParsers()
         if self.module is None:
             if len(argv) > 1 and argv[1].lower() == 'discover':
@@ -137,12 +142,15 @@
             self.testNames = (self.defaultTest,)
         else:
             self.testNames = list(self.defaultTest)
+        print('creating tests')
         self.createTests()
 
     def createTests(self):
         if self.testNames is None:
+            print('loading from module', self.module)
             self.test = self.testLoader.loadTestsFromModule(self.module)
         else:
+            print('loading', self.testNames, 'from', self.module)
             self.test = self.testLoader.loadTestsFromNames(self.testNames,
                                                            self.module)
 
@@ -228,6 +236,7 @@
         self.test = loader.discover(self.start, self.pattern, self.top)
 
     def runTests(self):
+        print('in runTests()')
         if self.catchbreak:
             installHandler()
         if self.testRunner is None:
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -110,6 +110,7 @@
     RegrTest('test_asdl_parser.py'),
     RegrTest('test_ast.py', core=True, usemodules='struct'),
     RegrTest('test_asynchat.py', usemodules='select fcntl'),
+    RegrTest('test_asyncio'),
     RegrTest('test_asyncore.py', usemodules='select fcntl'),
     RegrTest('test_atexit.py', core=True),
     RegrTest('test_audioop.py'),
@@ -199,7 +200,7 @@
     RegrTest('test_dynamic.py'),
     RegrTest('test_dynamicclassattribute.py'),
     RegrTest('test_eintr.py'),
-    RegrTest('test_email', skip="XXX is a directory"),
+    RegrTest('test_email'),
     RegrTest('test_ensurepip.py'),
     RegrTest('test_enum.py'),
     RegrTest('test_enumerate.py', core=True),
@@ -258,8 +259,9 @@
     RegrTest('test_idle.py'),
     RegrTest('test_imaplib.py'),
     RegrTest('test_imghdr.py'),
+    RegrTest('test_import'),
+    RegrTest('test_importlib', skip='XXX segfaults'),
     RegrTest('test_imp.py', core=True, usemodules='thread'),
-    RegrTest('test_importlib', 'XXX is a directory'),
     RegrTest('test_index.py'),
     RegrTest('test_inspect.py', usemodules="struct unicodedata"),
     RegrTest('test_int.py', core=True),
@@ -271,7 +273,7 @@
     RegrTest('test_iter.py', core=True),
     RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"),
     RegrTest('test_itertools.py', core=True, usemodules="itertools struct"),
-    RegrTest('test_json', skip="XXX is a directory"),
+    RegrTest('test_json'),
     RegrTest('test_keyword.py'),
     RegrTest('test_keywordonlyarg.py'),
     RegrTest('test_kqueue.py'),
@@ -437,9 +439,10 @@
     RegrTest('test_tix.py'),
     RegrTest('test_tk.py'),
     RegrTest('test_tokenize.py'),
-    RegrTest('test_trace.py'),
+    RegrTest('test_tools', skip="CPython internal details"),
     RegrTest('test_traceback.py', core=True),
     RegrTest('test_tracemalloc.py'),
+    RegrTest('test_trace.py'),
     RegrTest('test_ttk_guionly.py'),
     RegrTest('test_ttk_textonly.py'),
     RegrTest('test_tuple.py', core=True),
@@ -470,6 +473,7 @@
     RegrTest('test_venv.py', usemodules="struct"),
     RegrTest('test_wait3.py', usemodules="thread"),
     RegrTest('test_wait4.py', usemodules="thread"),
+    RegrTest('test_warnings'),
     RegrTest('test_wave.py'),
     RegrTest('test_weakref.py', core=True, usemodules='_weakref'),
     RegrTest('test_weakset.py'),
@@ -513,11 +517,23 @@
         cache[x.basename] = x
 
 def pytest_ignore_collect(path, config):
+    if path.basename == '__init__.py':
+        return False
     if path.isfile():
         regrtest = config._basename2spec.get(path.basename, None)
         if regrtest is None or path.dirpath() != testdir:
             return True
 
+def pytest_collect_file(path, parent):
+    if path.basename == '__init__.py':
+        # handle the RegrTest for the whole subpackage here
+        pkg_path = path.dirpath()
+        regrtest = parent.config._basename2spec.get(pkg_path.basename, None)
+        if pkg_path.dirpath() == testdir and regrtest:
+            return RunFileExternal(
+                pkg_path.basename, parent=parent, regrtest=regrtest)
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_pycollect_makemodule(path, parent):
     config = parent.config

From pypy.commits at gmail.com  Wed Nov 16 20:43:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 17:43:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 2to3
Message-ID: <582d0b5a.88711c0a.80263.5183@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88421:15a86b62540d
Date: 2016-11-17 01:42 +0000
http://bitbucket.org/pypy/pypy/changeset/15a86b62540d/

Log:	2to3

diff --git a/pypy/module/pypyjit/test_pypy_c/test_alloc.py b/pypy/module/pypyjit/test_pypy_c/test_alloc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_alloc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_alloc.py
@@ -16,7 +16,7 @@
                     N = %(size)d
                     part_a = 'a' * N
                     part_b = 'b' * N
-                    for i in xrange(20):
+                    for i in range(20):
                         ao = '%%s%%s' %% (part_a, part_b)
                     def main():
                         return 42
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -403,7 +403,7 @@
         def main(n):
             i = 1
             while i < n:
-                i += len(xrange(i+1)) - i
+                i += len(range(i+1)) - i
             return i
 
         log = self.run(main, [10000])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -401,7 +401,7 @@
             };
             """)
 
-            for i in xrange(n):
+            for i in range(n):
                 ffi.new("struct s *", [i, i, i])
 
         log = self.run(main, [300])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
--- a/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_jitlogparser.py
@@ -14,12 +14,12 @@
     def test(self):
         def fn_with_bridges(N):
             def is_prime(x):
-                for y in xrange(2, x):
+                for y in range(2, x):
                     if x % y == 0:
                         return False
                 return True
             result = 0
-            for x in xrange(N):
+            for x in range(N):
                 if x % 3 == 0:
                     result += 5
                 elif x % 5 == 0:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -179,7 +179,7 @@
         import codecs
 
         def main(n):
-            for i in xrange(n):
+            for i in range(n):
                 codecs.lookup('utf8')
             return i
         """, [1000])
@@ -197,7 +197,7 @@
     def test_decode_ascii(self):
         log = self.run("""
         def main(n):
-            for i in xrange(n):
+            for i in range(n):
                 unicode(str(i))
             return i
         """, [1000])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_weakref.py b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
--- a/pypy/module/pypyjit/test_pypy_c/test_weakref.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
@@ -14,7 +14,7 @@
 
         def main(n):
             obj = Dummy()
-            for i in xrange(n):
+            for i in range(n):
                 weakref.ref(obj, noop)
         """, [500])
         loop, = log.loops_by_filename(self.filepath)

From pypy.commits at gmail.com  Wed Nov 16 20:45:48 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 17:45:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: remove debugging leftovers
Message-ID: <582d0bcc.c4251c0a.beef3.58b0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88422:dcea68267797
Date: 2016-11-17 01:45 +0000
http://bitbucket.org/pypy/pypy/changeset/dcea68267797/

Log:	remove debugging leftovers

diff --git a/lib-python/3/test/test_importlib/__main__.py b/lib-python/3/test/test_importlib/__main__.py
--- a/lib-python/3/test/test_importlib/__main__.py
+++ b/lib-python/3/test/test_importlib/__main__.py
@@ -1,5 +1,4 @@
 from . import load_tests
-print('loaded tests')
 import unittest
 
 unittest.main()
diff --git a/lib-python/3/unittest/main.py b/lib-python/3/unittest/main.py
--- a/lib-python/3/unittest/main.py
+++ b/lib-python/3/unittest/main.py
@@ -59,7 +59,6 @@
                     testRunner=None, testLoader=loader.defaultTestLoader,
                     exit=True, verbosity=1, failfast=None, catchbreak=None,
                     buffer=None, warnings=None, *, tb_locals=False):
-        print('TestProgram()')
         if isinstance(module, str):
             self.module = __import__(module)
             for part in module.split('.')[1:]:
@@ -75,7 +74,6 @@
         self.verbosity = verbosity
         self.buffer = buffer
         self.tb_locals = tb_locals
-        print('1')
         if warnings is None and not sys.warnoptions:
             # even if DeprecationWarnings are ignored by default
             # print them anyway unless other warnings settings are
@@ -92,9 +90,7 @@
         self.testRunner = testRunner
         self.testLoader = testLoader
         self.progName = os.path.basename(argv[0])
-        print('parsing')
         self.parseArgs(argv)
-        print('running')
         self.runTests()
 
     def usageExit(self, msg=None):
@@ -115,7 +111,6 @@
             print(MODULE_EXAMPLES % {'prog': self.progName})
 
     def parseArgs(self, argv):
-        print('in parseArgs()')
         self._initArgParsers()
         if self.module is None:
             if len(argv) > 1 and argv[1].lower() == 'discover':
@@ -142,15 +137,12 @@
             self.testNames = (self.defaultTest,)
         else:
             self.testNames = list(self.defaultTest)
-        print('creating tests')
         self.createTests()
 
     def createTests(self):
         if self.testNames is None:
-            print('loading from module', self.module)
             self.test = self.testLoader.loadTestsFromModule(self.module)
         else:
-            print('loading', self.testNames, 'from', self.module)
             self.test = self.testLoader.loadTestsFromNames(self.testNames,
                                                            self.module)
 
@@ -236,7 +228,6 @@
         self.test = loader.discover(self.start, self.pattern, self.top)
 
     def runTests(self):
-        print('in runTests()')
         if self.catchbreak:
             installHandler()
         if self.testRunner is None:

From pypy.commits at gmail.com  Wed Nov 16 21:01:08 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 16 Nov 2016 18:01:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip hanging asyncio tests
Message-ID: <582d0f64.0a22c20a.3594d.2b59@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88423:c2a6c3eaf486
Date: 2016-11-17 02:00 +0000
http://bitbucket.org/pypy/pypy/changeset/c2a6c3eaf486/

Log:	skip hanging asyncio tests

diff --git a/lib-python/3/test/test_asyncio/test_events.py b/lib-python/3/test/test_asyncio/test_events.py
--- a/lib-python/3/test/test_asyncio/test_events.py
+++ b/lib-python/3/test/test_asyncio/test_events.py
@@ -909,6 +909,7 @@
         sslcontext = self._create_ssl_context(certfile, keyfile)
         return self._make_unix_server(factory, ssl=sslcontext)
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     def test_create_server_ssl(self):
         proto = MyProto(loop=self.loop)
@@ -945,6 +946,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_server_ssl()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
     def test_create_unix_server_ssl(self):
@@ -980,6 +982,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_unix_server_ssl()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     def test_create_server_ssl_verify_failed(self):
         proto = MyProto(loop=self.loop)
@@ -1013,6 +1016,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_server_ssl_verify_failed()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
     def test_create_unix_server_ssl_verify_failed(self):
@@ -1048,6 +1052,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_unix_server_ssl_verify_failed()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     def test_create_server_ssl_match_failed(self):
         proto = MyProto(loop=self.loop)
@@ -1080,6 +1085,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_server_ssl_match_failed()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
     def test_create_unix_server_ssl_verified(self):
@@ -1110,6 +1116,7 @@
         with test_utils.force_legacy_ssl_support():
             self.test_create_unix_server_ssl_verified()
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     @unittest.skipIf(ssl is None, 'No ssl module')
     def test_create_server_ssl_verified(self):
         proto = MyProto(loop=self.loop)
@@ -1139,6 +1146,7 @@
         server.close()
         self.loop.run_until_complete(proto.done)
 
+    @unittest.skipIf('__pypy__' in sys.modules, 'XXX: broken ssl')
     def test_legacy_create_server_ssl_verified(self):
         with test_utils.force_legacy_ssl_support():
             self.test_create_server_ssl_verified()

From pypy.commits at gmail.com  Thu Nov 17 03:47:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 Nov 2016 00:47:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: (arigo) when
 running 'py.test -A', detect complete crashes of CPython too
Message-ID: <582d6e8c.471ec20a.76ac6.9cc2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88424:106e9a48e6a9
Date: 2016-11-17 08:53 +0000
http://bitbucket.org/pypy/pypy/changeset/106e9a48e6a9/

Log:	(arigo) when running 'py.test -A', detect complete crashes of
	CPython too

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -172,7 +172,8 @@
         f.write('\n'.join(defs))
         f.write('def %s():\n' % target_name)
         f.write('\n'.join(source))
-        f.write("\n%s()\n" % target_name)
+        f.write("\ntry:\n    %s()\n" % target_name)
+        f.write('finally:\n    print("===aefwuiheawiu===")')
     helper_dir = os.path.join(pypydir, 'tool', 'cpyext')
     env = os.environ.copy()
     env['PYTHONPATH'] = helper_dir
@@ -186,6 +187,8 @@
                      (python_, usemodules))
     elif res > 0:
         raise AssertionError("Subprocess failed:\n" + stderr)
+    elif "===aefwuiheawiu===" not in stdout:
+        raise AssertionError("%r crashed:\n%s" % (python_, stderr))
 
 
 def extract_docstring_if_empty_function(fn):

From pypy.commits at gmail.com  Thu Nov 17 03:47:10 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 Nov 2016 00:47:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: (arigo) typos.
 write a minimal test now for the expected errors
Message-ID: <582d6e8e.54161c0a.ddac0.cb5d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88425:3f27a0c78aa5
Date: 2016-11-17 08:56 +0000
http://bitbucket.org/pypy/pypy/changeset/3f27a0c78aa5/

Log:	(arigo) typos. write a minimal test now for the expected errors

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -655,7 +655,9 @@
                 blockstack.append(addr)
                 blockstack.append(-1)
             elif op == POP_BLOCK:
-                assert len(blockstack) >= 3
+                if len(blockstack) < 3:
+                    raise oefmt(space.w_SystemError,
+                                "blocks not properly nested in this bytecode")
                 blockstack.pop()
                 setup_op = ord(code[blockstack.pop()])
                 if setup_op != SETUP_LOOP:
@@ -670,22 +672,24 @@
                 assert ii >= 0
                 handler_addr = blockstack[ii]
                 if addr == new_lasti:
-                    new_lasti_setup_addr = handler_addr
+                    new_lasti_handler_addr = handler_addr
                 if addr == self.last_instr:
-                    f_lasti_setup_addr = handler_addr
+                    f_lasti_handler_addr = handler_addr
 
             if op >= HAVE_ARGUMENT:
                 addr += 3
             else:
                 addr += 1
 
-        assert len(blockstack) == 0
+        if len(blockstack) != 1:
+            raise oefmt(space.w_SystemError,
+                        "blocks not properly nested in this bytecode")
 
         if new_lasti_handler_addr != f_lasti_handler_addr:
             raise oefmt(space.w_ValueError,
                         "can't jump into or out of an 'expect' or "
                         "'finally' block (%d -> %d)",
-                        f_lasti_handler_addr, new_lasti_setup_addr)
+                        f_lasti_handler_addr, new_lasti_handler_addr)
 
         # now we know we're not jumping into or out of a place which
         # needs a SysExcInfoRestorer.  Check that we're not jumping
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -87,6 +87,40 @@
         sys.settrace(None)
         # assert did not crash
 
+    def test_f_lineno_set_2(self):
+        counter = [0]
+        errors = []
+
+        def tracer(f, event, *args):
+            if event == 'line':
+                counter[0] += 1
+                if counter[0] == 2:
+                    try:
+                        f.f_lineno += 2
+                    except ValueError as e:
+                        errors.append(e)
+            return tracer
+
+        # obscure: call open beforehand, py3k's open invokes some app
+        # level code that confuses our tracing (likely due to the
+        # testing env, otherwise it's not a problem)
+        f = open(self.tempfile1, 'w')
+        def function():
+            try:
+                raise ValueError
+            except ValueError:
+                x = 42
+            return x
+
+        import sys
+        sys.settrace(tracer)
+        x = function()
+        sys.settrace(None)
+        assert x == 42
+        assert len(errors) == 1
+        assert str(errors[0]).startswith(
+            "can't jump into or out of an 'expect' or 'finally' block")
+
     def test_f_lineno_set_firstline(self):
         r"""
         seen = []

From pypy.commits at gmail.com  Thu Nov 17 03:49:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 00:49:24 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Document next cpython crasher
Message-ID: <582d6f14.ca06c20a.4b3f6.9791@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5748:572cc735325d
Date: 2016-11-17 09:49 +0100
http://bitbucket.org/pypy/extradoc/changeset/572cc735325d/

Log:	Document next cpython crasher

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -41,6 +41,29 @@
   if chain=False.  This can rarely cause random nonsense in the main
   program.
 
+* setting f_lineno didn't evolve when the rest of the bytecodes evolved,
+  which means it is not safe any more::
+
+    import sys
+
+    def f():
+        try:
+            raise ValueError    # line 5
+        except ValueError:
+            print(42)           # line 7
+
+    def my_trace(*args):
+        print(args)
+        if args[1] == 'line':
+            f = args[0]
+            if f.f_lineno == 5:
+                f.f_lineno = 7
+        return my_trace
+
+    sys.settrace(my_trace)
+    f()
+    sys.settrace(None)
+
 
 Other bugs
 ----------

From pypy.commits at gmail.com  Thu Nov 17 04:17:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 01:17:16 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix (probably a merge error?)
Message-ID: <582d759c.471ec20a.76ac6.a8e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88426:31fb79a2893f
Date: 2016-11-17 10:16 +0100
http://bitbucket.org/pypy/pypy/changeset/31fb79a2893f/

Log:	Fix (probably a merge error?)

diff --git a/pytest.py b/pytest.py
--- a/pytest.py
+++ b/pytest.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # PYTHON_ARGCOMPLETE_OK
 """
 pytest: unit and functional testing with Python.

From pypy.commits at gmail.com  Thu Nov 17 07:34:52 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 04:34:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: copy over certificates of cpython
 stdlib (ssl + ssl_tests) target 3.5.2
Message-ID: <582da3ec.d32f1c0a.29acc.20b9@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88427:742c85b008ab
Date: 2016-11-17 10:05 +0100
http://bitbucket.org/pypy/pypy/changeset/742c85b008ab/

Log:	copy over certificates of cpython stdlib (ssl + ssl_tests) target
	3.5.2

diff --git a/lib-python/3/test/capath/0e4015b9.0 b/lib-python/3/test/capath/0e4015b9.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/0e4015b9.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/capath/ce7b8643.0 b/lib-python/3/test/capath/ce7b8643.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/ce7b8643.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/selfsigned_pythontestdotnet.pem b/lib-python/3/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/3/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/3/test/selfsigned_pythontestdotnet.pem
@@ -1,5 +1,5 @@
 -----BEGIN CERTIFICATE-----
-MIIChzCCAfCgAwIBAgIJAKGU95wKR8pSMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
 IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
 bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
@@ -8,9 +8,9 @@
 aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
 Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
 Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjKTAnMCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MA0GCSqGSIb3DQEBBQUAA4GBAIOXmdtM
-eG9qzP9TiXW/Gc/zI4cBfdCpC+Y4gOfC9bQUC7hefix4iO3+iZjgy3X/FaRxUUoV
-HKiXcXIaWqTSUWp45cSh0MbwZXudp6JIAptzdAhvvCrPKeC9i9GvxsPD4LtDAL97
-vSaxQBezA7hdxZd90/EeyMgVZgAnTCnvAWX9
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
 -----END CERTIFICATE-----
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -23,9 +23,6 @@
 
 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES)
 HOST = support.HOST
-IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL')
-IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0)
-
 
 def data_file(*name):
     return os.path.join(os.path.dirname(__file__), *name)
@@ -57,8 +54,6 @@
 SIGNED_CERTFILE = data_file("keycert3.pem")
 SIGNED_CERTFILE2 = data_file("keycert4.pem")
 SIGNING_CA = data_file("pycacert.pem")
-# cert with all kinds of subject alt names
-ALLSANFILE = data_file("allsans.pem")
 
 REMOTE_HOST = "self-signed.pythontest.net"
 REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem")
@@ -148,8 +143,8 @@
     def test_str_for_enums(self):
         # Make sure that the PROTOCOL_* constants have enum-like string
         # reprs.
-        proto = ssl.PROTOCOL_TLS
-        self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS')
+        proto = ssl.PROTOCOL_SSLv23
+        self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_SSLv23')
         ctx = ssl.SSLContext(proto)
         self.assertIs(ctx.protocol, proto)
 
@@ -281,27 +276,6 @@
 
         self.assertEqual(p['subjectAltName'], san)
 
-    def test_parse_all_sans(self):
-        p = ssl._ssl._test_decode_cert(ALLSANFILE)
-        self.assertEqual(p['subjectAltName'],
-            (
-                ('DNS', 'allsans'),
-                ('othername', '<unsupported>'),
-                ('othername', '<unsupported>'),
-                ('email', 'user at example.org'),
-                ('DNS', 'www.example.org'),
-                ('DirName',
-                    ((('countryName', 'XY'),),
-                    (('localityName', 'Castle Anthrax'),),
-                    (('organizationName', 'Python Software Foundation'),),
-                    (('commonName', 'dirname example'),))),
-                ('URI', 'https://www.python.org/'),
-                ('IP Address', '127.0.0.1'),
-                ('IP Address', '0:0:0:0:0:0:0:1\n'),
-                ('Registered ID', '1.2.3.4.5')
-            )
-        )
-
     def test_DER_to_PEM(self):
         with open(CAFILE_CACERT, 'r') as f:
             pem = f.read()
@@ -338,8 +312,8 @@
         self.assertGreaterEqual(status, 0)
         self.assertLessEqual(status, 15)
         # Version string as returned by {Open,Libre}SSL, the format might change
-        if IS_LIBRESSL:
-            self.assertTrue(s.startswith("LibreSSL {:d}".format(major)),
+        if "LibreSSL" in s:
+            self.assertTrue(s.startswith("LibreSSL {:d}.{:d}".format(major, minor)),
                             (s, t, hex(n)))
         else:
             self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)),
@@ -816,8 +790,7 @@
     def test_constructor(self):
         for protocol in PROTOCOLS:
             ssl.SSLContext(protocol)
-        ctx = ssl.SSLContext()
-        self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS)
+        self.assertRaises(TypeError, ssl.SSLContext)
         self.assertRaises(ValueError, ssl.SSLContext, -1)
         self.assertRaises(ValueError, ssl.SSLContext, 42)
 
@@ -838,15 +811,15 @@
     def test_options(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
-        default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
-        if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0):
-            default |= ssl.OP_NO_COMPRESSION
-        self.assertEqual(default, ctx.options)
+        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3,
+                         ctx.options)
         ctx.options |= ssl.OP_NO_TLSv1
-        self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options)
+        self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1,
+                         ctx.options)
         if can_clear_options():
-            ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1)
-            self.assertEqual(default, ctx.options)
+            ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1
+            self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
+                             ctx.options)
             ctx.options = 0
             # Ubuntu has OP_NO_SSLv3 forced on by default
             self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3)
@@ -1182,7 +1155,6 @@
         self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH')
 
     @unittest.skipIf(sys.platform == "win32", "not-Windows specific")
-    @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars")
     def test_load_default_certs_env(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with support.EnvironmentVarGuard() as env:
@@ -1778,13 +1750,13 @@
             sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNotNone(sslobj.shared_ciphers())
+            self.assertIsNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNotNone(sslobj.shared_ciphers())
+            self.assertIsNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
@@ -1898,14 +1870,7 @@
                 else:
                     self.sock.close()
 
-            # PyPy change
             def run(self):
-                try:
-                    self._run()
-                finally:
-                    self.close()
-
-            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
@@ -2668,7 +2633,7 @@
                     s.close()
 
         def test_socketserver(self):
-            """Using socketserver to create and manage SSL connections."""
+            """Using a SocketServer to create and manage SSL connections."""
             server = make_https_server(self, certfile=CERTFILE)
             # try to connect
             if support.verbose:
@@ -2695,6 +2660,8 @@
 
         def test_asyncore_server(self):
             """Check the example asyncore integration."""
+            indata = "TEST MESSAGE of mixed case\n"
+
             if support.verbose:
                 sys.stdout.write("\n")
 
@@ -2826,13 +2793,20 @@
                         # consume data
                         s.read()
 
+                data = b"data"
+
                 # read(-1, buffer) is supported, even though read(-1) is not
-                data = b"data"
                 s.send(data)
                 buffer = bytearray(len(data))
                 self.assertEqual(s.read(-1, buffer), len(data))
                 self.assertEqual(buffer, data)
 
+                # recv/read(0) should return no data
+                s.send(data)
+                self.assertEqual(s.recv(0), b"")
+                self.assertEqual(s.read(0), b"")
+                self.assertEqual(s.read(), data)
+
                 # Make sure sendmsg et al are disallowed to avoid
                 # inadvertent disclosure of data and/or corruption
                 # of the encrypted data stream
@@ -2848,26 +2822,6 @@
 
                 s.close()
 
-        def test_recv_zero(self):
-            server = ThreadedEchoServer(CERTFILE)
-            server.__enter__()
-            self.addCleanup(server.__exit__, None, None)
-            s = socket.create_connection((HOST, server.port))
-            self.addCleanup(s.close)
-            s = ssl.wrap_socket(s, suppress_ragged_eofs=False)
-            self.addCleanup(s.close)
-
-            # recv/read(0) should return no data
-            s.send(b"data")
-            self.assertEqual(s.recv(0), b"")
-            self.assertEqual(s.read(0), b"")
-            self.assertEqual(s.read(), b"data")
-
-            # Should not block if the other end sends no data
-            s.setblocking(False)
-            self.assertEqual(s.recv(0), b"")
-            self.assertEqual(s.recv_into(bytearray()), 0)
-
         def test_nonblocking_send(self):
             server = ThreadedEchoServer(CERTFILE,
                                         certreqs=ssl.CERT_NONE,
@@ -2889,10 +2843,8 @@
                 # will be full and the call will block
                 buf = bytearray(8192)
                 def fill_buffer():
-                    i = 0
                     while True:
                         s.send(buf)
-                        i += 1
                 self.assertRaises((ssl.SSLWantWriteError,
                                    ssl.SSLWantReadError), fill_buffer)
 
@@ -3028,7 +2980,7 @@
                 with context.wrap_socket(socket.socket()) as s:
                     self.assertIs(s.version(), None)
                     s.connect((HOST, server.port))
-                    self.assertEqual(s.version(), 'TLSv1')
+                    self.assertEqual(s.version(), "TLSv1")
                 self.assertIs(s.version(), None)
 
         @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL")
@@ -3170,36 +3122,24 @@
                 (['http/3.0', 'http/4.0'], None)
             ]
             for client_protocols, expected in protocol_tests:
-                server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+                server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 server_context.load_cert_chain(CERTFILE)
                 server_context.set_alpn_protocols(server_protocols)
-                client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+                client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                 client_context.load_cert_chain(CERTFILE)
                 client_context.set_alpn_protocols(client_protocols)
-
-                try:
-                    stats = server_params_test(client_context,
-                                               server_context,
-                                               chatty=True,
-                                               connectionchatty=True)
-                except ssl.SSLError as e:
-                    stats = e
-
-                if expected is None and IS_OPENSSL_1_1:
-                    # OpenSSL 1.1.0 raises handshake error
-                    self.assertIsInstance(stats, ssl.SSLError)
-                else:
-                    msg = "failed trying %s (s) and %s (c).\n" \
-                        "was expecting %s, but got %%s from the %%s" \
-                            % (str(server_protocols), str(client_protocols),
-                                str(expected))
-                    client_result = stats['client_alpn_protocol']
-                    self.assertEqual(client_result, expected,
-                                     msg % (client_result, "client"))
-                    server_result = stats['server_alpn_protocols'][-1] \
-                        if len(stats['server_alpn_protocols']) else 'nothing'
-                    self.assertEqual(server_result, expected,
-                                     msg % (server_result, "server"))
+                stats = server_params_test(client_context, server_context,
+                                           chatty=True, connectionchatty=True)
+
+                msg = "failed trying %s (s) and %s (c).\n" \
+                      "was expecting %s, but got %%s from the %%s" \
+                          % (str(server_protocols), str(client_protocols),
+                             str(expected))
+                client_result = stats['client_alpn_protocol']
+                self.assertEqual(client_result, expected, msg % (client_result, "client"))
+                server_result = stats['server_alpn_protocols'][-1] \
+                    if len(stats['server_alpn_protocols']) else 'nothing'
+                self.assertEqual(server_result, expected, msg % (server_result, "server"))
 
         def test_selected_npn_protocol(self):
             # selected_npn_protocol() is None unless NPN is used
@@ -3347,23 +3287,13 @@
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
-                client_context.set_ciphers("AES128:AES256")
-                server_context.set_ciphers("AES256")
-                alg1 = "AES256"
-                alg2 = "AES-256"
-            else:
-                client_context.set_ciphers("AES:3DES")
-                server_context.set_ciphers("3DES")
-                alg1 = "3DES"
-                alg2 = "DES-CBC3"
-
+            client_context.set_ciphers("RC4")
+            server_context.set_ciphers("AES:RC4")
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                if not alg1 in name.split("-") and alg2 not in name:
-                    self.fail(name)
+                self.assertIn("RC4", name.split("-"))
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
diff --git a/lib_pypy/openssl/_cffi_src/openssl/err.py b/lib_pypy/openssl/_cffi_src/openssl/err.py
--- a/lib_pypy/openssl/_cffi_src/openssl/err.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/err.py
@@ -196,9 +196,34 @@
 static const int SSL_TLSEXT_ERR_ALERT_FATAL;
 static const int SSL_TLSEXT_ERR_NOACK;
 
+static const int SSL_AD_CLOSE_NOTIFY;
+static const int SSL_AD_UNEXPECTED_MESSAGE;
+static const int SSL_AD_BAD_RECORD_MAC;
+static const int SSL_AD_RECORD_OVERFLOW;
+static const int SSL_AD_DECOMPRESSION_FAILURE;
+static const int SSL_AD_HANDSHAKE_FAILURE;
+static const int SSL_AD_BAD_CERTIFICATE;
+static const int SSL_AD_UNSUPPORTED_CERTIFICATE;
+static const int SSL_AD_CERTIFICATE_REVOKED;
+static const int SSL_AD_CERTIFICATE_EXPIRED;
+static const int SSL_AD_CERTIFICATE_UNKNOWN;
+static const int SSL_AD_ILLEGAL_PARAMETER;
+static const int SSL_AD_UNKNOWN_CA;
+static const int SSL_AD_ACCESS_DENIED;
+static const int SSL_AD_DECODE_ERROR;
+static const int SSL_AD_DECRYPT_ERROR;
+static const int SSL_AD_PROTOCOL_VERSION;
+static const int SSL_AD_INSUFFICIENT_SECURITY;
 static const int SSL_AD_INTERNAL_ERROR;
-static const int SSL_AD_ACCESS_DENIED;
-static const int SSL_AD_HANDSHAKE_FAILURE;
+static const int SSL_AD_USER_CANCELLED;
+static const int SSL_AD_NO_RENEGOTIATION;
+
+static const int SSL_AD_UNSUPPORTED_EXTENSION;
+static const int SSL_AD_CERTIFICATE_UNOBTAINABLE;
+static const int SSL_AD_UNRECOGNIZED_NAME;
+static const int SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE;
+static const int SSL_AD_BAD_CERTIFICATE_HASH_VALUE;
+static const int SSL_AD_UNKNOWN_PSK_IDENTITY;
 """
 
 FUNCTIONS = """
@@ -255,4 +280,24 @@
 static const long Cryptography_HAS_RSA_R_PKCS_DECODING_ERROR = 0;
 static const long RSA_R_PKCS_DECODING_ERROR = 0;
 #endif
+
+#ifndef SSL_AD_UNSUPPORTED_EXTENSION
+    static const int SSL_AD_UNSUPPORTED_EXTENSION = -1;
+#endif
+#ifndef SSL_AD_CERTIFICATE_UNOBTAINABLE
+    static const int SSL_AD_CERTIFICATE_UNOBTAINABLE = -1;
+#endif
+#ifndef SSL_AD_UNRECOGNIZED_NAME
+    static const int SSL_AD_UNRECOGNIZED_NAME = -1;
+#endif
+#ifndef SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE
+    static const int SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE = -1;
+#endif
+#ifndef SSL_AD_BAD_CERTIFICATE_HASH_VALUE
+    static const int SSL_AD_BAD_CERTIFICATE_HASH_VALUE = -1;
+#endif
+#ifndef SSL_AD_UNKNOWN_PSK_IDENTITY
+    static const int SSL_AD_UNKNOWN_PSK_IDENTITY = -1;
+#endif
+
 """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -695,9 +695,9 @@
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1;
 
 #ifdef OPENSSL_NO_TLSEXT
+static const long Cryptography_NO_TLSEXT = 1;
+#else
 static const long Cryptography_NO_TLSEXT = 0;
-#else
-static const long Cryptography_NO_TLSEXT = 1;
 #endif
 
 #ifdef OPENSSL_NPN_NEGOTIATED
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -8,7 +8,7 @@
 from openssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from openssl._stdssl.utility import (_str_with_len, _bytes_with_len,
-    _str_to_ffi_buffer, _str_from_buf)
+    _str_to_ffi_buffer, _str_from_buf, _cstr_decode_fs)
 from openssl._stdssl.error import (ssl_error, pyssl_error,
         SSLError, SSLZeroReturnError, SSLWantReadError,
         SSLWantWriteError, SSLSyscallError,
@@ -17,8 +17,9 @@
         SSL_ERROR_SSL, SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE,
         SSL_ERROR_WANT_X509_LOOKUP, SSL_ERROR_SYSCALL,
         SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_CONNECT,
-        SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE)
-
+        SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE,
+        pyerr_write_unraisable)
+from openssl._stdssl import error
 
 OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
 OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER
@@ -85,6 +86,12 @@
 else:
     CHANNEL_BINDING_TYPES = []
 
+for name in error.SSL_AD_NAMES:
+    lib_attr = 'SSL_AD_' + name
+    attr = 'ALERT_DESCRIPTION_' + name
+    if hasattr(lib, lib_attr):
+        globals()[attr] = getattr(lib, lib_attr)
+
 # init open ssl
 lib.SSL_load_error_strings()
 lib.SSL_library_init()
@@ -103,6 +110,7 @@
     callable = None
     password = None
     operationerror = None
+    handle = None
 PWINFO_STORAGE = {}
 
 def _Cryptography_pem_password_cb(buf, size, rwflag, userdata):
@@ -258,8 +266,8 @@
         self._owner = None
         self.server_hostname = None
         self.socket = None
-        self.alpn_protocols = ffi.NULL
-        self.npn_protocols = ffi.NULL
+        #self.alpn_protocols = ffi.NULL
+        #self.npn_protocols = ffi.NULL
 
     @property
     def owner(self):
@@ -710,7 +718,7 @@
 class _SSLContext(object):
     __slots__ = ('ctx', '_check_hostname', 'servername_callback',
                  'alpn_protocols', 'npn_protocols', 'set_hostname',
-                 '_set_hostname_handle')
+                 '_set_hostname_handle', '_npn_protocols_handle')
 
     def __new__(cls, protocol):
         self = object.__new__(cls)
@@ -861,8 +869,6 @@
         pw_info = PasswordInfo()
         index = -1
         if password is not None:
-            index = _thread.get_ident()
-            PWINFO_STORAGE[index] = pw_info
 
             if callable(password):
                 pw_info.callable = password
@@ -872,9 +878,11 @@
                 else:
                     raise TypeError("password should be a string or callable")
 
-            handle = ffi.new_handle(pw_info) # XXX MUST NOT be garbage collected
+            pw_info.handle = ffi.new_handle(pw_info)
+            index = _thread.get_ident()
+            PWINFO_STORAGE[index] = pw_info
             lib.SSL_CTX_set_default_passwd_cb(self.ctx, Cryptography_pem_password_cb)
-            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, handle)
+            lib.SSL_CTX_set_default_passwd_cb_userdata(self.ctx, pw_info.handle)
 
         try:
             ffi.errno = 0
@@ -908,7 +916,7 @@
 
             ret = lib.SSL_CTX_check_private_key(self.ctx)
             if ret != 1:
-                raise _ssl_seterror(None, -1)
+                raise ssl_error(None)
         finally:
             if index >= 0:
                 del PWINFO_STORAGE[index]
@@ -1033,6 +1041,7 @@
         return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
 
 
+#    REVIEW, how to do that properly
 #    def _finalize_(self):
 #        ctx = self.ctx
 #        if ctx:
@@ -1125,8 +1134,8 @@
             return
         if not callable(callback):
             raise TypeError("not a callable object")
-        self.set_hostname = callback
-        self._set_hostname_handle = ffi.new_handle(self)
+        scb = ServernameCallback(callback, self)
+        self._set_hostname_handle = ffi.new_handle(scb)
         lib.Cryptography_SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
         lib.Cryptography_SSL_CTX_set_tlsext_servername_arg(self.ctx, self._set_hostname_handle)
 
@@ -1146,6 +1155,7 @@
         if HAS_NPN:
             self.npn_protocols = ffi.from_buffer(protos)
             handle = ffi.new_handle(self)
+            self._npn_protocols_handle = handle # track a reference to the handle
             lib.SSL_CTX_set_next_protos_advertised_cb(self.ctx, advertise_npn_callback, handle)
             lib.SSL_CTX_set_next_proto_select_cb(self.ctx, select_npn_callback, handle)
         else:
@@ -1156,20 +1166,21 @@
 if HAS_SNI and not lib.Cryptography_NO_TLSEXT:
     @ffi.callback("int(SSL*,int*,void*)")
     def _servername_callback(s, al, arg):
-        ssl_ctx = ffi.from_handle(arg)
+        scb = ffi.from_handle(arg)
+        ssl_ctx = scb.ctx
         servername = lib.SSL_get_servername(s, lib.TLSEXT_NAMETYPE_host_name)
-    #ifdef WITH_THREAD
-        # TODO PyGILState_STATE gstate = PyGILState_Ensure();
-    #endif
+        set_hostname = scb.callback
+        #ifdef WITH_THREAD
+            # TODO PyGILState_STATE gstate = PyGILState_Ensure();
+        #endif
 
-        if ssl_ctx.set_hostname is None:
+        if set_hostname is None:
             #/* remove race condition in this the call back while if removing the
             # * callback is in progress */
-    #ifdef WITH_THREAD
-            # TODO PyGILState_Release(gstate);
-    #endif
+            #ifdef WITH_THREAD
+                    # TODO PyGILState_Release(gstate);
+            #endif
             return lib.SSL_TLSEXT_ERR_OK
-        #}
 
         ssl = ffi.from_handle(lib.SSL_get_app_data(s))
         assert isinstance(ssl, _SSLSocket)
@@ -1192,10 +1203,9 @@
 
         if servername == ffi.NULL:
             try:
-                result = ssl_ctx.set_hostname(ssl_socket, None, ssl_ctx)
-            except:
-                # TODO
-                #        PyErr_WriteUnraisable(ssl_ctx->set_hostname);
+                result = set_hostname(ssl_socket, None, ssl_ctx)
+            except Exception as e:
+                pyerr_write_unraisable(e, set_hostname)
                 al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
                 return lib.SSL_TLSEXT_ERR_ALERT_FATAL
         else:
@@ -1203,31 +1213,44 @@
 
             try:
                 servername_idna = servername.decode("idna")
-            except UnicodeDecodeError:
-                raise # TODO?
-    #            PyErr_WriteUnraisable(servername_o);
+            except UnicodeDecodeError as e:
+                pyerr_write_unraisable(e, servername)
 
             try:
-                result = ssl_ctx.set_hostname(ssl_socket, servername_idna, ssl_ctx)
-            except:
-                # TODO
-                #        PyErr_WriteUnraisable(ssl_ctx->set_hostname);
+                result = set_hostname(ssl_socket, servername_idna, ssl_ctx)
+            except Exception as e:
+                pyerr_write_unraisable(e, set_hostname)
                 al[0] = lib.SSL_AD_HANDSHAKE_FAILURE
                 return lib.SSL_TLSEXT_ERR_ALERT_FATAL
 
         if result is not None:
+            # this is just a poor man's emulation:
+            # in CPython this works a bit different, it calls all the way
+            # down from PyLong_AsLong to _PyLong_FromNbInt which raises
+            # a TypeError if there is no nb_int slot filled.
             try:
-                al[0] = int(result)
-            except:
-    #                PyErr_WriteUnraisable(result);
-               al[0] = lib.SSL_AD_INTERNAL_ERROR
-               return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+                if isinstance(result, int):
+                    al[0] = result
+                else:
+                    if result is not None:
+                        if hasattr(result,'__int__'):
+                            al[0] = result.__int__()
+                            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
+                    # needed because sys.exec_info is used in pyerr_write_unraisable
+                    raise TypeError("an integer is required (got type %s)" % result)
+            except TypeError as e:
+                pyerr_write_unraisable(e, result)
+                al[0] = lib.SSL_AD_INTERNAL_ERROR
+            return lib.SSL_TLSEXT_ERR_ALERT_FATAL
         else:
             # TODO gil state release?
             return lib.SSL_TLSEXT_ERR_OK
 
 class ServernameCallback(object):
-    ctx = None
+    def __init__(self, callback, ctx):
+        self.callback = callback
+        self.ctx = ctx
+
 SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
 TEST = None
 
@@ -1287,13 +1310,13 @@
         """Whether the memory BIO is at EOF."""
         return lib.BIO_ctrl_pending(self.bio) == 0 and self.eof_written
 
-    def write(self, _bytes):
+    def write(self, strlike):
         INT_MAX = 2**31-1
-        if isinstance(_bytes, memoryview):
-            # REVIEW pypy does not support from_buffer of a memoryview
-            # copies the data!
-            _bytes = bytes(_bytes)
-        buf = ffi.from_buffer(_bytes)
+        if isinstance(strlike, memoryview):
+            # FIXME pypy must support get_raw_address for
+            # StringBuffer to remove this case!
+            strlike = strlike.tobytes()
+        buf = ffi.from_buffer(strlike)
         if len(buf) > INT_MAX:
             raise OverflowError("string longer than %d bytes", INT_MAX)
 
@@ -1346,7 +1369,7 @@
         ok = lib.RAND_bytes(buf, count)
         if ok == 1:
             return ffi.string(buf)
-    raise ssl_error("", errcode=lib.ERR_get_error())
+    raise ssl_error(None, errcode=lib.ERR_get_error())
 
 def RAND_pseudo_bytes(count):
     return _RAND_bytes(count, True)
@@ -1358,19 +1381,6 @@
     buf = _str_to_ffi_buffer(view)
     lib.RAND_add(buf, len(buf), entropy)
 
-
-def _cstr_decode_fs(buf):
-#define CONVERT(info, target) { \
-#        const char *tmp = (info); \
-#        target = NULL; \
-#        if (!tmp) { Py_INCREF(Py_None); target = Py_None; } \
-#        else if ((target = PyUnicode_DecodeFSDefault(tmp)) == NULL) { \
-#            target = PyBytes_FromString(tmp); } \
-#        if (!target) goto error; \
-#    }
-    # REVIEW
-    return ffi.string(buf).decode(sys.getfilesystemencoding())
-
 def get_default_verify_paths():
 
     ofile_env = _cstr_decode_fs(lib.X509_get_default_cert_file_env())
@@ -1385,7 +1395,6 @@
     odir = _cstr_decode_fs(lib.X509_get_default_cert_dir())
     if odir is None:
         return odir
-
     return (ofile_env, ofile, odir_env, odir);
 
 @ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
@@ -1395,28 +1404,28 @@
                                  ffi.cast("unsigned char*",ctx.alpn_protocols), len(ctx.alpn_protocols),
                                  client_protocols, client_protocols_len)
 
- at ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
-def select_npn_callback(ssl, out, outlen, server_protocols, server_protocols_len, args):
-    ctx = ffi.from_handle(args)
-    return do_protocol_selection(0, out, outlen, server_protocols, server_protocols_len,
-                                 ffi.cast("unsigned char*",ctx.npn_protocols), len(ctx.npn_protocols))
+if lib.Cryptography_HAS_NPN_NEGOTIATED:
+    @ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
+    def select_npn_callback(ssl, out, outlen, server_protocols, server_protocols_len, args):
+        ctx = ffi.from_handle(args)
+        return do_protocol_selection(0, out, outlen, server_protocols, server_protocols_len,
+                                     ffi.cast("unsigned char*",ctx.npn_protocols), len(ctx.npn_protocols))
 
 
- at ffi.callback("int(SSL*,const unsigned char**, unsigned int*, void*)")
-def advertise_npn_callback(ssl, data, length, args):
-    ctx = ffi.from_handle(args)
+    @ffi.callback("int(SSL*,const unsigned char**, unsigned int*, void*)")
+    def advertise_npn_callback(ssl, data, length, args):
+        ctx = ffi.from_handle(args)
 
-    if not ctx.npn_protocols:
-        data[0] = ffi.new("unsigned char*", b"")
-        length[0] = 0
-    else:
-        data[0] = ffi.cast("unsigned char*",ctx.npn_protocols)
-        length[0] = len(ctx.npn_protocols)
+        if not ctx.npn_protocols:
+            data[0] = ffi.new("unsigned char*", b"")
+            length[0] = 0
+        else:
+            data[0] = ffi.cast("unsigned char*",ctx.npn_protocols)
+            length[0] = len(ctx.npn_protocols)
 
-    return lib.SSL_TLSEXT_ERR_OK
+        return lib.SSL_TLSEXT_ERR_OK
 
 
-if lib.Cryptography_HAS_NPN_NEGOTIATED:
     def do_protocol_selection(alpn, out, outlen, server_protocols, server_protocols_len,
                                                  client_protocols, client_protocols_len):
         if client_protocols == ffi.NULL:
diff --git a/lib_pypy/openssl/_stdssl/error.py b/lib_pypy/openssl/_stdssl/error.py
--- a/lib_pypy/openssl/_stdssl/error.py
+++ b/lib_pypy/openssl/_stdssl/error.py
@@ -1,3 +1,5 @@
+import sys
+import traceback
 from _openssl import ffi
 from _openssl import lib
 
@@ -46,7 +48,7 @@
     if errstr is None:
         errcode = lib.ERR_peek_last_error()
     try:
-        return fill_sslerror(SSLError, errcode, errstr)
+        return fill_sslerror(SSLError, errcode, errstr, errcode)
     finally:
         lib.ERR_clear_error()
 
@@ -136,6 +138,7 @@
         lib_str = LIB_CODES_TO_NAMES.get(err_lib, None)
         if errstr is None:
             errstr = _str_from_buf(lib.ERR_reason_error_string(errcode))
+    msg = errstr
     if not errstr:
         msg = "unknown error"
     if reason_str and lib_str:
@@ -148,3 +151,49 @@
     err_value.library = lib_str if lib_str else None
     return err_value
 
+def pyerr_write_unraisable(exc, obj):
+    f = sys.stderr
+
+    if obj:
+        f.write("Exception ignored in: ")
+        f.write(repr(obj))
+        f.write("\n")
+
+    t, v, tb = sys.exc_info()
+    traceback.print_tb(tb, file=f)
+
+    assert isinstance(v, Exception)
+    f.write(t.__module__ + "." + t.__name__)
+    f.write(": ")
+    f.write(str(v))
+    f.write("\n")
+
+SSL_AD_NAMES = [
+    "ACCESS_DENIED",
+    "BAD_CERTIFICATE",
+    "BAD_CERTIFICATE_HASH_VALUE",
+    "BAD_CERTIFICATE_STATUS_RESPONSE",
+    "BAD_RECORD_MAC",
+    "CERTIFICATE_EXPIRED",
+    "CERTIFICATE_REVOKED",
+    "CERTIFICATE_UNKNOWN",
+    "CERTIFICATE_UNOBTAINABLE",
+    "CLOSE_NOTIFY",
+    "DECODE_ERROR",
+    "DECOMPRESSION_FAILURE",
+    "DECRYPT_ERROR",
+    "HANDSHAKE_FAILURE",
+    "ILLEGAL_PARAMETER",
+    "INSUFFICIENT_SECURITY",
+    "INTERNAL_ERROR",
+    "NO_RENEGOTIATION",
+    "PROTOCOL_VERSION",
+    "RECORD_OVERFLOW",
+    "UNEXPECTED_MESSAGE",
+    "UNKNOWN_CA",
+    "UNKNOWN_PSK_IDENTITY",
+    "UNRECOGNIZED_NAME",
+    "UNSUPPORTED_CERTIFICATE",
+    "UNSUPPORTED_EXTENSION",
+    "USER_CANCELLED",
+]
diff --git a/lib_pypy/openssl/_stdssl/utility.py b/lib_pypy/openssl/_stdssl/utility.py
--- a/lib_pypy/openssl/_stdssl/utility.py
+++ b/lib_pypy/openssl/_stdssl/utility.py
@@ -1,3 +1,4 @@
+import sys
 from _openssl import ffi
 from _openssl import lib
 
@@ -36,3 +37,15 @@
 def _str_from_buf(buf):
     return ffi.string(buf).decode('utf-8')
 
+def _cstr_decode_fs(buf):
+#define CONVERT(info, target) { \
+#        const char *tmp = (info); \
+#        target = NULL; \
+#        if (!tmp) { Py_INCREF(Py_None); target = Py_None; } \
+#        else if ((target = PyUnicode_DecodeFSDefault(tmp)) == NULL) { \
+#            target = PyBytes_FromString(tmp); } \
+#        if (!target) goto error; \
+#    }
+    # REVIEW
+    return ffi.string(buf).decode(sys.getfilesystemencoding())
+

From pypy.commits at gmail.com  Thu Nov 17 07:34:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 04:34:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: passing all tests in test_ssl.py
Message-ID: <582da3ee.581d1c0a.580fd.2aba@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88428:d9f42756c388
Date: 2016-11-17 13:34 +0100
http://bitbucket.org/pypy/pypy/changeset/d9f42756c388/

Log:	passing all tests in test_ssl.py

diff --git a/lib_pypy/openssl/_cffi_src/openssl/bio.py b/lib_pypy/openssl/_cffi_src/openssl/bio.py
--- a/lib_pypy/openssl/_cffi_src/openssl/bio.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/bio.py
@@ -82,6 +82,8 @@
 int BIO_write(BIO *, const void *, int);
 int BIO_puts(BIO *, const char *);
 int BIO_method_type(const BIO *);
+
+int * Cryptography_bio_references(const BIO *);
 """
 
 MACROS = """
@@ -132,7 +134,12 @@
 long BIO_set_nbio(BIO *, long);
 void BIO_set_retry_read(BIO *);
 void BIO_clear_retry_flags(BIO *);
+
+#define CRYPTO_LOCK_BIO ...
 """
 
 CUSTOMIZATIONS = """
+int * Cryptography_bio_references(const BIO * b) {
+    return &b->references;
+}
 """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/crypto.py b/lib_pypy/openssl/_cffi_src/openssl/crypto.py
--- a/lib_pypy/openssl/_cffi_src/openssl/crypto.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/crypto.py
@@ -57,6 +57,8 @@
 
 /* This was removed in 1.1.0 */
 void CRYPTO_lock(int, int, const char *, int);
+
+void CRYPTO_add(void*,int,int);
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/openssl/_stdssl/__init__.py b/lib_pypy/openssl/_stdssl/__init__.py
--- a/lib_pypy/openssl/_stdssl/__init__.py
+++ b/lib_pypy/openssl/_stdssl/__init__.py
@@ -98,6 +98,10 @@
 # TODO threads?
 lib.OpenSSL_add_all_algorithms()
 
+def check_signals():
+    # TODO PyErr_CheckSignal equivalent for pypy?
+    pass
+
 def _socket_timeout(s):
     if s is None:
         return 0.0
@@ -218,13 +222,14 @@
         if sock:
             lib.SSL_set_fd(ssl, sock.fileno())
         else:
-            raise NotImplementedError("implement _SSLSocket inbio, outbio params")
-            # /* BIOs are reference counted and SSL_set_bio borrows our reference.
-            #  * To prevent a double free in memory_bio_dealloc() we need to take an
-            #  * extra reference here. */
-            # CRYPTO_add(&inbio->bio->references, 1, CRYPTO_LOCK_BIO);
-            # CRYPTO_add(&outbio->bio->references, 1, CRYPTO_LOCK_BIO);
-            # SSL_set_bio(self->ssl, inbio->bio, outbio->bio);
+            # BIOs are reference counted and SSL_set_bio borrows our reference.
+            # To prevent a double free in memory_bio_dealloc() we need to take an
+            # extra reference here.
+            irefaddr = lib.Cryptography_bio_references(inbio.bio);
+            orefaddr = lib.Cryptography_bio_references(outbio.bio);
+            lib.CRYPTO_add(irefaddr, 1, lib.CRYPTO_LOCK_BIO)
+            lib.CRYPTO_add(orefaddr, 1, lib.CRYPTO_LOCK_BIO)
+            lib.SSL_set_bio(self.ssl, inbio.bio, outbio.bio)
 
         mode = lib.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER
         if lib.SSL_MODE_AUTO_RETRY:
@@ -287,12 +292,17 @@
 
     @context.setter
     def context(self, value):
-        self.ctx = value
+        if isinstance(value, _SSLContext):
+            if not HAS_SNI:
+                raise NotImplementedError("setting a socket's "
+                        "context is not supported by your OpenSSL library")
+            self.ctx = value
+            lib.SSL_set_SSL_CTX(self.ssl, self.ctx.ctx);
+        else:
+            raise TypeError("The value must be a SSLContext")
 
     def do_handshake(self):
-        sock = self.get_socket_or_None()
-        if sock is None:
-            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+        sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
         timeout = _socket_timeout(sock)
         if sock:
@@ -313,8 +323,7 @@
             err = lib.SSL_get_error(ssl, ret)
             # end allow threads
 
-            #if (PyErr_CheckSignals())
-            #    goto error;
+            check_signals()
 
             if has_timeout:
                 # REIVIEW monotonic clock?
@@ -399,8 +408,7 @@
             err = lib.SSL_get_error(self.ssl, length)
             #PySSL_END_ALLOW_THREADS
 
-            # TODO if (PyErr_CheckSignals())
-            # TODO     goto error;
+            check_signals()
 
             if has_timeout:
                 # TODO monotonic clock
@@ -428,14 +436,12 @@
             raise pyssl_error(self, length)
 
     def read(self, length, buffer_into=None):
-        sock = self.get_socket_or_None()
         ssl = self.ssl
 
         if length < 0 and buffer_into is None:
             raise ValueError("size should not be negative")
 
-        if sock is None:
-            raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
+        sock = self.get_socket_or_connection_gone()
 
         if not buffer_into:
             dest = _buffer_new(length)
@@ -467,9 +473,7 @@
             err = lib.SSL_get_error(self.ssl, count);
             #PySSL_END_ALLOW_THREADS
 
-            # TODO
-            #if (PyErr_CheckSignals())
-            #    goto error;
+            check_signals()
 
             if has_timeout:
                 timeout = deadline - time.time() # TODO ? _PyTime_GetMonotonicClock();
@@ -512,9 +516,10 @@
 
     def shared_ciphers(self):
         sess = lib.SSL_get_session(self.ssl)
-
+        if sess == ffi.NULL:
+            return None
         ciphers = lib.Cryptography_get_ssl_session_ciphers(sess)
-        if sess is None or ciphers == ffi.NULL:
+        if ciphers == ffi.NULL:
             return None
         res = []
         count = lib.sk_SSL_CIPHER_num(ciphers)
@@ -559,18 +564,28 @@
         return self.socket()
 
     def get_socket_or_connection_gone(self):
+        """ There are three states:
+            1) self.socket is None (In C that would mean: self->Socket == NULL)
+            2) self.socket() is None (-> The socket is gone)
+            3) self.socket() is not None
+            This method returns True if there is not weakref object allocated
+        """
         if self.socket is None:
+            return None
+        sock = self.socket()
+        if not sock:
             raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
-        return self.socket()
+        return sock
 
     def shutdown(self):
         sock = self.get_socket_or_None()
         nonblocking = False
         ssl = self.ssl
 
-        if sock is not None:
+        if self.socket is not None:
             # Guard against closed socket
-            if sock.fileno() < 0:
+            sock = self.socket()
+            if sock is None or sock.fileno() < 0:
                 raise ssl_error("Underlying socket connection gone", SSL_ERROR_NO_SOCKET)
 
             timeout = _socket_timeout(sock)
@@ -1129,7 +1144,6 @@
                     "is not in the current OpenSSL library.")
         if callback is None:
             lib.SSL_CTX_set_tlsext_servername_callback(self.ctx, ffi.NULL)
-            self.set_hostname = None
             self._set_hostname_handle = None
             return
         if not callable(callback):
@@ -1161,6 +1175,16 @@
         else:
             raise NotImplementedError("The NPN extension requires OpenSSL 1.0.1 or later.")
 
+    def _wrap_bio(self, incoming, outgoing, server_side, server_hostname):
+        # server_hostname is either None (or absent), or to be encoded
+        # using the idna encoding.
+        hostname = None
+        if server_hostname is not None:
+            hostname = server_hostname.encode("idna")
+
+        sock = _SSLSocket._new__ssl_socket(self, None, server_side, hostname, incoming, outgoing)
+        return sock
+
 
 
 if HAS_SNI and not lib.Cryptography_NO_TLSEXT:
@@ -1360,15 +1384,16 @@
 def _RAND_bytes(count, pseudo):
     if count < 0:
         raise ValueError("num must be positive")
-    buf = ffi.new("unsigned char[]", b"\x00"*count)
+    buf = ffi.new("unsigned char[%d]" % count)
     if pseudo:
         ok = lib.RAND_pseudo_bytes(buf, count)
         if ok == 1 or ok == 0:
-            return (ffi.string(buf), ok == 1)
+            _bytes = _bytes_with_len(buf, count)
+            return (_bytes, ok == 1)
     else:
         ok = lib.RAND_bytes(buf, count)
         if ok == 1:
-            return ffi.string(buf)
+            return _bytes_with_len(buf, count)
     raise ssl_error(None, errcode=lib.ERR_get_error())
 
 def RAND_pseudo_bytes(count):
diff --git a/lib_pypy/openssl/_stdssl/error.py b/lib_pypy/openssl/_stdssl/error.py
--- a/lib_pypy/openssl/_stdssl/error.py
+++ b/lib_pypy/openssl/_stdssl/error.py
@@ -101,14 +101,18 @@
             errval = SSL_ERROR_WANT_CONNECT
         elif err == SSL_ERROR_SYSCALL:
             if e == 0:
-                if ret == 0 or obj.get_socket_or_None() is None:
-                    errtype = EOFError
+                if ret == 0 or obj.socket is not None:
+                    errtype = SSLEOFError
                     errstr = "EOF occurred in violation of protocol"
                     errval = SSL_ERROR_EOF
-                elif ret == -1:
+                elif ret == -1 and obj.socket is not None:
                     # the underlying BIO reported an I/0 error
-                    errno = ffi.errno
-                    return IOError(errno)
+                    lib.ERR_clear_error()
+                    s = obj.get_socket_or_None()
+                    s.errorhandler()
+                    assert 0, "must not get here"
+                    #errno = ffi.errno
+                    #return IOError(errno)
                 else:
                     errtype = SSLSyscallError
                     errstr = "Some I/O error occurred"

From pypy.commits at gmail.com  Thu Nov 17 07:45:57 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 04:45:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: some simplifications, removed comments
Message-ID: <582da685.41a3c20a.7f663.fd8c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88429:d4d6dd1d67d1
Date: 2016-11-17 13:45 +0100
http://bitbucket.org/pypy/pypy/changeset/d4d6dd1d67d1/

Log:	some simplifications, removed comments

diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -248,8 +248,6 @@
 Cryptography_STACK_OF_X509_NAME *SSL_load_client_CA_file(const char *);
 
 const char *SSL_get_servername(const SSL *, const int);
-//long SSL_CTX_callback_ctrl(SSL_CTX *, int, void (*)(void));
-//long SSL_CTX_ctrl(SSL_CTX *, int, long, void *);
 """
 
 MACROS = """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509v3.py b/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509v3.py
@@ -183,12 +183,9 @@
 GENERAL_NAMES *GENERAL_NAMES_new(void);
 void GENERAL_NAMES_free(GENERAL_NAMES *);
 void *X509V3_EXT_d2i(X509_EXTENSION *);
-/* X509_get_ext_d2i is already defined, there might be a better solution
-   to expose it to the lib object?  */
-void * _X509_get_ext_d2i(X509 *, int, int *, int *);
 /* X509 is private, there is no way to access the field crldp other than
    adding it to the typedef or expose a function like this: */
-Cryptography_STACK_OF_DIST_POINT * _X509_get_crldp(const X509 *);
+Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 *);
 int X509_check_ca(X509 *);
 """
 
@@ -302,15 +299,17 @@
 DIST_POINT_NAME *DIST_POINT_NAME_new(void);
 void DIST_POINT_NAME_free(DIST_POINT_NAME *);
 
+void * X509_get_ext_d2i(const X509 *, int, int *, int *);
 """
 
 CUSTOMIZATIONS = """
-void * _X509_get_ext_d2i(X509 * x, int nid, int * a, int * b) {
-    return X509_get_ext_d2i(x, nid, a, b);
+#if OPENSSL_VERSION_NUMBER >= 0x10001000L
+Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 * x) {
+    return x->crldp;
 }
-#if OPENSSL_VERSION_NUMBER >= 0x10001000L
-Cryptography_STACK_OF_DIST_POINT * _X509_get_crldp(const X509 * x) {
-    return x->crldp;
+#else
+Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 * x) {
+    return NULL;
 }
 #endif
 """
diff --git a/lib_pypy/openssl/_stdssl/certificate.py b/lib_pypy/openssl/_stdssl/certificate.py
--- a/lib_pypy/openssl/_stdssl/certificate.py
+++ b/lib_pypy/openssl/_stdssl/certificate.py
@@ -26,7 +26,7 @@
     return (name, value)
 
 def _get_aia_uri(certificate, nid):
-    info = lib._X509_get_ext_d2i(certificate, lib.NID_info_access, ffi.NULL, ffi.NULL)
+    info = lib.X509_get_ext_d2i(certificate, lib.NID_info_access, ffi.NULL, ffi.NULL)
     if (info == ffi.NULL):
         return None;
     if lib.sk_ACCESS_DESCRIPTION_num(info) == 0:
@@ -244,16 +244,12 @@
 
 
 def _get_crl_dp(certificate):
-#    STACK_OF(DIST_POINT) *dps;
-#    int i, j;
-#    PyObject *lst, *res = NULL;
-#
     if lib.OPENSSL_VERSION_NUMBER < 0x10001000:
         dps = lib.X509_get_ext_d2i(certificate, lib.NID_crl_distribution_points, ffi.NULL, ffi.NULL)
     else:
         # Calls x509v3_cache_extensions and sets up crldp
         lib.X509_check_ca(certificate)
-        dps = lib._X509_get_crldp(certificate)
+        dps = lib.Cryptography_X509_get_crldp(certificate)
     if dps is ffi.NULL:
         return None
 

From pypy.commits at gmail.com  Thu Nov 17 08:15:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 05:15:23 -0800 (PST)
Subject: [pypy-commit] pypy clean-exported-state: Close branch
 clean-exported-state
Message-ID: <582dad6b.46bb1c0a.3750c.268c@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: clean-exported-state
Changeset: r88430:c01760e8436e
Date: 2016-11-17 13:14 +0000
http://bitbucket.org/pypy/pypy/changeset/c01760e8436e/

Log:	Close branch clean-exported-state


From pypy.commits at gmail.com  Thu Nov 17 08:15:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 05:15:41 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in clean-exported-state (pull
 request #490)
Message-ID: <582dad7d.12921c0a.9ece6.e80b@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r88431:ab8a0b092d2d
Date: 2016-11-17 13:14 +0000
http://bitbucket.org/pypy/pypy/changeset/ab8a0b092d2d/

Log:	Merged in clean-exported-state (pull request #490)

	Clean exported state

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -108,7 +108,7 @@
     """
     log_noopt = False
 
-    def __init__(self, trace, celltoken, state,
+    def __init__(self, trace, celltoken, state, runtime_boxes,
                  call_pure_results=None, enable_opts=None,
                  inline_short_preamble=True):
         self.trace = trace
@@ -117,6 +117,8 @@
         self.state = state
         self.call_pure_results = call_pure_results
         self.inline_short_preamble = inline_short_preamble
+        assert runtime_boxes is not None
+        self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -124,7 +126,11 @@
         assert unroll # we should not be here if it's disabled
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.call_pure_results, self.inline_short_preamble)
+            self.runtime_boxes, self.call_pure_results, self.inline_short_preamble)
+
+    def forget_optimization_info(self):
+        self.state.forget_optimization_info()
+        CompileData.forget_optimization_info(self)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
@@ -290,7 +296,7 @@
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     jitcell_token.target_tokens = [start_descr]
-    loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -360,7 +366,7 @@
     history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
     enable_opts = jitdriver_sd.warmstate.enable_opts
     call_pure_results = metainterp.call_pure_results
-    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -372,6 +378,7 @@
         history.cut(cut)
         history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
         loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+                                     jumpargs,
                                      call_pure_results=call_pure_results,
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
@@ -516,7 +523,7 @@
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
-        #     somehoe for ContinueRunningNormally
+        #     somehow for ContinueRunningNormally
         if reset_values:
             item.reset_value()
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
      AbstractResOp, GuardResOp
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, we_are_debug
 from rpython.jit.metainterp.optimizeopt import info
         
 
@@ -172,7 +172,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getfield(descr, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.AbstractStructPtrInfo):
                 assert opinfo in self.cached_infos
         if isinstance(res, PreambleOp):
@@ -202,7 +202,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getitem(descr, self.index, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.ArrayPtrInfo):
                 assert opinfo in self.cached_infos
         if (isinstance(res, PreambleOp) and
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,9 +24,20 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
+def check_no_forwarding(lsts):
+    for lst in lsts:
+        for op in lst:
+            assert op.get_forwarded() is None
+
 class LoopInfo(object):
     label_op = None
 
+    def _check_no_forwarding(self):
+        pass
+
+    def forget_optimization_info(self):
+        pass
+
 class BasicLoopInfo(LoopInfo):
     def __init__(self, inputargs, quasi_immutable_deps, jump_op):
         self.inputargs = inputargs
@@ -555,7 +566,8 @@
         return (BasicLoopInfo(trace.inputargs, self.quasi_immutable_deps, last_op),
                 self._newoperations)
 
-    def _clean_optimization_info(self, lst):
+    @staticmethod
+    def _clean_optimization_info(lst):
         for op in lst:
             if op.get_forwarded() is not None:
                 op.set_forwarded(None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -5,6 +5,7 @@
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
+from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 
 class PreambleOp(AbstractResOp):
@@ -18,7 +19,7 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     op = None
-    
+
     def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
@@ -51,7 +52,13 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
-    pass
+    res = None
+
+    def _check_no_forwarding(self):
+        assert self.res.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.res.clear_forwarded()
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -101,6 +108,14 @@
                                        descr=sop.getdescr())
         return ProducedShortOp(self, preamble_op)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.getfield_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.getfield_op.clear_forwarded()
+
     def __repr__(self):
         return "HeapOp(%r)" % (self.res,)
 
@@ -193,6 +208,16 @@
                 l.append(pop)
         return l
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        self.one._check_no_forwarding()
+        self.two._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.one.forget_optimization_info()
+        self.two.forget_optimization_info()
+
     def repr(self, memo):
         return "CompoundOp(%s, %s, %s)" % (self.res.repr(memo),
                                            self.one.repr(memo),
@@ -203,7 +228,7 @@
 
 class ProducedShortOp(AbstractProducedShortOp):
     invented_name = False
-    
+
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
@@ -215,6 +240,14 @@
     def repr(self, memo):
         return self.short_op.repr(memo)
 
+    def _check_no_forwarding(self):
+        self.short_op._check_no_forwarding()
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.short_op.forget_optimization_info()
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
 
@@ -235,6 +268,14 @@
     def repr(self, memo):
         return "INP(%s)" % (self.res.repr(memo),)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
@@ -454,16 +495,23 @@
         self.sb = sb
         self.extra_same_as = self.sb.extra_same_as
         self.target_token = target_token
+        self.build_inplace = False
 
     def setup(self, jump_args, short, label_args):
         self.jump_args = jump_args
         self.short = short
         self.label_args = label_args
+        self.build_inplace = True
 
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
         """
+        # Could this be considered a speculative error?
+        # This check should only fail when trying to jump to an existing trace
+        # by forcing portions of the virtualstate.
+        if not self.build_inplace:
+            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         op = preamble_op.op.get_box_replacement()
         if preamble_op.invented_name:
             self.extra_same_as.append(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -573,6 +573,7 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
+        state[0]._check_no_forwarding()
         return state
 
     def _convert_call_pure_results(self, d):
@@ -621,7 +622,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
         loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, call_pure_results)
+            celltoken, start_state, runtime_boxes, call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimize import InvalidLoop, SpeculativeError
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo
+     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo, check_no_forwarding
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
@@ -35,7 +35,7 @@
 
     def setinfo_from_preamble_list(self, lst, infos):
         for item in lst:
-            if item is None:
+            if item is None or isinstance(item, Const):
                 continue
             i = infos.get(item, None)
             if i is not None:
@@ -97,7 +97,6 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
-
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
@@ -115,26 +114,22 @@
         return modifier.get_virtual_state(args)
 
     def _check_no_forwarding(self, lsts, check_newops=True):
-        for lst in lsts:
-            for op in lst:
-                assert op.get_forwarded() is None
+        check_no_forwarding(lsts)
         if check_newops:
             assert not self.optimizer._newoperations
 
-
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs,
-                                           runtime_boxes, memo)
+                                           info.inputargs, memo)
         exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state,
+    def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
                              call_pure_results, inline_short_preamble=True):
         trace = trace.get_iter()
         try:
@@ -186,7 +181,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
+                    end_jump, label_op, runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -199,7 +194,7 @@
             # to the preamble.
             try:
                 new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+                        end_jump, label_op, runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
@@ -276,8 +271,7 @@
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, runtime_boxes,
-                                           box_names_memo)
+                                           info.inputargs, box_names_memo)
         exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
@@ -440,8 +434,7 @@
                 continue
             self._expand_info(item, infos)
 
-    def export_state(self, original_label_args, renamed_inputargs,
-                     runtime_boxes, memo):
+    def export_state(self, original_label_args, renamed_inputargs, memo):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
         self.optimizer.flush()
@@ -462,16 +455,17 @@
             op = produced_op.short_op.res
             if not isinstance(op, Const):
                 self._expand_info(op, infos)
-        self.optimizer._clean_optimization_info(end_args)
         return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
-                             short_inputargs, runtime_boxes, memo)
+                             short_inputargs, memo)
 
     def import_state(self, targetargs, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
                 len(targetargs))
+        self._check_no_forwarding([targetargs])
+        exported_state._check_no_forwarding()
         for i, target in enumerate(exported_state.next_iteration_args):
             source = targetargs[i]
             assert source is not target
@@ -527,13 +521,11 @@
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
     * quasi_immutable_deps - for tracking quasi immutables
-    * runtime_boxes - runtime values for boxes, necessary when generating
-                      guards to jump to
     """
 
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
-                 short_inputargs, runtime_boxes, memo):
+                 short_inputargs, memo):
         self.end_args = end_args
         self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
@@ -541,8 +533,8 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
-        self.runtime_boxes = runtime_boxes
         self.dump(memo)
+        self.forget_optimization_info()
 
     def dump(self, memo):
         if have_debug_prints():
@@ -552,5 +544,35 @@
                 debug_print("  " + box.repr(memo))
             debug_stop("jit-log-exported-state")
 
+    def _check_no_forwarding(self):
+        """ Ensures that no optimization state is attached to relevant operations
+        before importing anything. """
+        # Some of these may be redunant
+        check_no_forwarding([
+            self.end_args,
+            self.next_iteration_args,
+            self.renamed_inputargs,
+            self.short_inputargs,
+            self.exported_infos.keys()])
+        for box in self.short_boxes:
+            box._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        """ Clean up optimization info on all operations stored in the ExportedState.
+
+        This function needs to be called when exporting the optimizer state to
+        prevent leaking of optimization information between invocations of the
+        optimizer.
+
+        That includes cleaning up in the event that optimize_peeled_loop() fails
+        with an InvalidLoop exception, as optimize_peeled_loop() mutates the
+        contents of ExportedState.
+        """
+        Optimizer._clean_optimization_info(self.renamed_inputargs)
+        for box in self.exported_infos.iterkeys():
+            box.clear_forwarded()
+        for box in self.short_boxes:
+            box.forget_optimization_info()
+
     def final(self):
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1976,6 +1976,8 @@
         self.aborted_tracing_greenkey = None
 
     def retrace_needed(self, trace, exported_state):
+        if not we_are_translated():
+            exported_state._check_no_forwarding()
         self.partial_trace = trace
         self.retracing_from = self.potential_retrace_position
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -52,6 +52,10 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
+    def clear_forwarded(self):
+        if self.get_forwarded() is not None:
+            self.set_forwarded(None)
+
     @specialize.arg(1)
     def get_box_replacement(op, not_const=False):
         # Read the chain "op, op._forwarded, op._forwarded._forwarded..."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -295,10 +295,15 @@
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
 _translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
+_rpy_assert_value = CDefinedIntSymbolic('RPY_ASSERT_VALUE', default=1)
 
 def we_are_translated_to_c():
     return we_are_translated() and _translated_to_c
 
+def we_are_debug():
+    """ Returns True when not translated or translated with debugging enabled. """
+    return not we_are_translated() or (_translated_to_c and _rpy_assert_value)
+
 # ____________________________________________________________
 
 def instantiate(cls, nonmovable=False):
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -31,8 +31,10 @@
 RPY_EXTERN
 void RPyAssertFailed(const char* filename, long lineno,
                      const char* function, const char *msg);
+#  define RPY_ASSERT_VALUE 1
 #else
 #  define RPyAssert(x, msg)   /* nothing */
+#  define RPY_ASSERT_VALUE 0
 #endif
 
 RPY_EXTERN

From pypy.commits at gmail.com  Thu Nov 17 08:18:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 05:18:47 -0800 (PST)
Subject: [pypy-commit] pypy default: document
Message-ID: <582dae37.c515c20a.cc3bd.0b8c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88432:dc3941998986
Date: 2016-11-17 10:20 +0100
http://bitbucket.org/pypy/pypy/changeset/dc3941998986/

Log:	document

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -14,3 +14,5 @@
 .. branch: union-side-effects-2
 
 Try to improve the consistency of RPython annotation unions.
+
+.. branch: pytest-2.9.2

From pypy.commits at gmail.com  Thu Nov 17 08:18:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 05:18:49 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <582dae39.44941c0a.3c83f.36a8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88433:5990b8c98760
Date: 2016-11-17 14:17 +0100
http://bitbucket.org/pypy/pypy/changeset/5990b8c98760/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -14,3 +14,7 @@
 .. branch: union-side-effects-2
 
 Try to improve the consistency of RPython annotation unions.
+
+.. branch: clean-exported-state
+
+Clean-ups in the jit optimizeopt

From pypy.commits at gmail.com  Thu Nov 17 08:18:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 05:18:51 -0800 (PST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <582dae3b.88711c0a.80263.375f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88434:3dc381df7878
Date: 2016-11-17 14:18 +0100
http://bitbucket.org/pypy/pypy/changeset/3dc381df7878/

Log:	merge heads

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -15,6 +15,8 @@
 
 Try to improve the consistency of RPython annotation unions.
 
+.. branch: pytest-2.9.2
+
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt

From pypy.commits at gmail.com  Thu Nov 17 08:19:47 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 05:19:47 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: copy changes made to cryptography,
 rename method name of call site
Message-ID: <582dae73.51ce190a.41205.3128@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88435:74ee14214a4e
Date: 2016-11-17 14:18 +0100
http://bitbucket.org/pypy/pypy/changeset/74ee14214a4e/

Log:	copy changes made to cryptography, rename method name of call site

diff --git a/lib_pypy/openssl/_cffi_src/build_openssl.py b/lib_pypy/openssl/_cffi_src/build_openssl.py
--- a/lib_pypy/openssl/_cffi_src/build_openssl.py
+++ b/lib_pypy/openssl/_cffi_src/build_openssl.py
@@ -7,7 +7,7 @@
 import os
 import sys
 
-from openssl._cffi_src.utils import (
+from _cffi_src.utils import (
     build_ffi_for_binding, compiler_type, extra_link_args
 )
 
diff --git a/lib_pypy/openssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/openssl/_cffi_src/hazmat_src/padding.c
--- a/lib_pypy/openssl/_cffi_src/hazmat_src/padding.c
+++ b/lib_pypy/openssl/_cffi_src/hazmat_src/padding.c
@@ -4,25 +4,25 @@
 
 /* Returns the value of the input with the most-significant-bit copied to all
    of the bits. */
-static uint8_t Cryptography_DUPLICATE_MSB_TO_ALL(uint8_t a) {
-    return (1 - (a >> (sizeof(uint8_t) * 8 - 1))) - 1;
+static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) {
+    return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1;
 }
 
-/* This returns 0xFF if a < b else 0x00, but does so in a constant time
+/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time
    fashion */
-static uint8_t Cryptography_constant_time_lt(uint8_t a, uint8_t b) {
+static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) {
     a -= b;
     return Cryptography_DUPLICATE_MSB_TO_ALL(a);
 }
 
 uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
-                                         uint8_t block_len) {
-    uint8_t i;
-    uint8_t pad_size = data[block_len - 1];
-    uint8_t mismatch = 0;
+                                         uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
     for (i = 0; i < block_len; i++) {
         unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
-        uint8_t b = data[block_len - 1 - i];
+        uint16_t b = data[block_len - 1 - i];
         mismatch |= (mask & (pad_size ^ b));
     }
 
@@ -31,6 +31,7 @@
     mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
 
     /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
     mismatch |= mismatch >> 4;
     mismatch |= mismatch >> 2;
     mismatch |= mismatch >> 1;
@@ -39,14 +40,14 @@
 }
 
 uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
-                                            uint8_t block_len) {
-    uint8_t i;
-    uint8_t pad_size = data[block_len - 1];
-    uint8_t mismatch = 0;
+                                            uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
     /* Skip the first one with the pad size */
     for (i = 1; i < block_len; i++) {
         unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
-        uint8_t b = data[block_len - 1 - i];
+        uint16_t b = data[block_len - 1 - i];
         mismatch |= (mask & b);
     }
 
@@ -55,6 +56,7 @@
     mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
 
     /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
     mismatch |= mismatch >> 4;
     mismatch |= mismatch >> 2;
     mismatch |= mismatch >> 1;
diff --git a/lib_pypy/openssl/_cffi_src/openssl/callbacks.py b/lib_pypy/openssl/_cffi_src/openssl/callbacks.py
--- a/lib_pypy/openssl/_cffi_src/openssl/callbacks.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/callbacks.py
@@ -12,6 +12,9 @@
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/x509_vfy.h>
+#include <openssl/crypto.h>
+
+#include <pythread.h>
 """
 
 TYPES = """
@@ -37,6 +40,7 @@
 """
 
 FUNCTIONS = """
+int _setup_ssl_threads(void);
 """
 
 MACROS = """
@@ -50,4 +54,71 @@
     # backwards compatibility for old cffi version on PyPy
     # and Python >=3.5 (https://github.com/pyca/cryptography/issues/2970)
     TYPES = "static const long Cryptography_STATIC_CALLBACKS;"
-    CUSTOMIZATIONS = "static const long Cryptography_STATIC_CALLBACKS = 0;"
+    CUSTOMIZATIONS = """static const long Cryptography_STATIC_CALLBACKS = 0;
+"""
+
+CUSTOMIZATIONS += """
+/* This code is derived from the locking code found in the Python _ssl module's
+   locking callback for OpenSSL.
+
+   Copyright 2001-2016 Python Software Foundation; All Rights Reserved.
+*/
+
+static unsigned int _ssl_locks_count = 0;
+static PyThread_type_lock *_ssl_locks = NULL;
+
+static void _ssl_thread_locking_function(int mode, int n, const char *file,
+                                         int line) {
+    /* this function is needed to perform locking on shared data
+       structures. (Note that OpenSSL uses a number of global data
+       structures that will be implicitly shared whenever multiple
+       threads use OpenSSL.) Multi-threaded applications will
+       crash at random if it is not set.
+
+       locking_function() must be able to handle up to
+       CRYPTO_num_locks() different mutex locks. It sets the n-th
+       lock if mode & CRYPTO_LOCK, and releases it otherwise.
+
+       file and line are the file number of the function setting the
+       lock. They can be useful for debugging.
+    */
+
+    if ((_ssl_locks == NULL) ||
+        (n < 0) || ((unsigned)n >= _ssl_locks_count)) {
+        return;
+    }
+
+    if (mode & CRYPTO_LOCK) {
+        PyThread_acquire_lock(_ssl_locks[n], 1);
+    } else {
+        PyThread_release_lock(_ssl_locks[n]);
+    }
+}
+
+int _setup_ssl_threads(void) {
+    unsigned int i;
+
+    if (_ssl_locks == NULL) {
+        _ssl_locks_count = CRYPTO_num_locks();
+        _ssl_locks = PyMem_New(PyThread_type_lock, _ssl_locks_count);
+        if (_ssl_locks == NULL) {
+            PyErr_NoMemory();
+            return 0;
+        }
+        memset(_ssl_locks, 0, sizeof(PyThread_type_lock) * _ssl_locks_count);
+        for (i = 0;  i < _ssl_locks_count;  i++) {
+            _ssl_locks[i] = PyThread_allocate_lock();
+            if (_ssl_locks[i] == NULL) {
+                unsigned int j;
+                for (j = 0;  j < i;  j++) {
+                    PyThread_free_lock(_ssl_locks[j]);
+                }
+                PyMem_Free(_ssl_locks);
+                return 0;
+            }
+        }
+        CRYPTO_set_locking_callback(_ssl_thread_locking_function);
+    }
+    return 1;
+}
+"""
diff --git a/lib_pypy/openssl/_cffi_src/openssl/ssl.py b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/openssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/ssl.py
@@ -137,16 +137,14 @@
 typedef ... SSL_METHOD;
 typedef ... SSL_CTX;
 
-
 typedef ... SSL_SESSION;
 typedef ... SSL;
 
-typedef ... Cryptography_STACK_OF_SSL_CIPHER;
-
 static const long TLSEXT_NAMETYPE_host_name;
 
+typedef ... SSL_CIPHER;
+typedef ... Cryptography_STACK_OF_SSL_CIPHER;
 typedef ... COMP_METHOD;
-typedef ... SSL_CIPHER;
 """
 
 FUNCTIONS = """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509_vfy.py
@@ -181,7 +181,7 @@
                                     Cryptography_STACK_OF_ASN1_OBJECT *);
 void X509_VERIFY_PARAM_set_depth(X509_VERIFY_PARAM *, int);
 int X509_VERIFY_PARAM_get_depth(const X509_VERIFY_PARAM *);
-
+void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *);
 int Cryptography_X509_OBJECT_get_type(const X509_OBJECT *);
 X509 * Cryptography_X509_OBJECT_data_x509(X509_OBJECT*);
 """
diff --git a/lib_pypy/openssl/_cffi_src/openssl/x509name.py b/lib_pypy/openssl/_cffi_src/openssl/x509name.py
--- a/lib_pypy/openssl/_cffi_src/openssl/x509name.py
+++ b/lib_pypy/openssl/_cffi_src/openssl/x509name.py
@@ -35,7 +35,7 @@
 int X509_NAME_get_index_by_NID(X509_NAME *, int, int);
 int X509_NAME_cmp(const X509_NAME *, const X509_NAME *);
 X509_NAME *X509_NAME_dup(X509_NAME *);
-int X509_NAME_ENTRY_set(const X509_NAME_ENTRY * ne);
+int Cryptography_X509_NAME_ENTRY_set(X509_NAME_ENTRY *);
 """
 
 MACROS = """
@@ -77,7 +77,13 @@
 """
 
 CUSTOMIZATIONS = """
-int X509_NAME_ENTRY_set(const X509_NAME_ENTRY * ne) {
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+int Cryptography_X509_NAME_ENTRY_set(X509_NAME_ENTRY *ne) {
+    return X509_NAME_ENTRY_set(ne);
+}
+#else
+int Cryptography_X509_NAME_ENTRY_set(X509_NAME_ENTRY *ne) {
     return ne->set;
 }
+#endif
 """
diff --git a/lib_pypy/openssl/_stdssl/certificate.py b/lib_pypy/openssl/_stdssl/certificate.py
--- a/lib_pypy/openssl/_stdssl/certificate.py
+++ b/lib_pypy/openssl/_stdssl/certificate.py
@@ -139,7 +139,7 @@
         entry = lib.X509_NAME_get_entry(xname, index_counter);
 
         # check to see if we've gotten to a new RDN
-        _set = lib.X509_NAME_ENTRY_set(entry)
+        _set = lib.Cryptography_X509_NAME_ENTRY_set(entry)
         if rdn_level >= 0:
             if rdn_level != _set:
                 dn.append(tuple(rdn))

From pypy.commits at gmail.com  Thu Nov 17 08:19:50 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 05:19:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: remove the old pypy/module/_ssl module
Message-ID: <582dae76.1a4b2e0a.d6bae.2f1a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88436:c22e66d77f66
Date: 2016-11-17 14:19 +0100
http://bitbucket.org/pypy/pypy/changeset/c22e66d77f66/

Log:	remove the old pypy/module/_ssl module

diff too long, truncating to 2000 out of 3517 lines

diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
deleted file mode 100644
--- a/pypy/module/_ssl/__init__.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import sys
-from rpython.rlib.rarithmetic import intmask
-from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._ssl import ssl_data
-
-class Module(MixedModule):
-    """Implementation module for SSL socket operations.
-    See the socket module for documentation."""
-
-    interpleveldefs = {
-        '_test_decode_cert': 'interp_ssl._test_decode_cert',
-        'txt2obj': 'interp_ssl.txt2obj',
-        'nid2obj': 'interp_ssl.nid2obj',
-        'get_default_verify_paths': 'interp_ssl.get_default_verify_paths',
-
-        'SSLError': 'interp_ssl.get_error(space).w_error',
-        'SSLZeroReturnError': 'interp_ssl.get_error(space).w_ZeroReturnError',
-        'SSLWantReadError': 'interp_ssl.get_error(space).w_WantReadError',
-        'SSLWantWriteError': 'interp_ssl.get_error(space).w_WantWriteError',
-        'SSLEOFError': 'interp_ssl.get_error(space).w_EOFError',
-        'SSLSyscallError': 'interp_ssl.get_error(space).w_SyscallError',
-
-        '_SSLSocket': 'interp_ssl.SSLSocket',
-        '_SSLContext': 'interp_ssl.SSLContext',
-        'MemoryBIO': 'interp_ssl.MemoryBIO',
-    }
-
-    if sys.platform == 'win32':
-        interpleveldefs['enum_certificates'] = 'interp_win32.enum_certificates_w'
-        interpleveldefs['enum_crls'] = 'interp_win32.enum_crls_w'
-
-    appleveldefs = {
-    }
-
-    @classmethod
-    def buildloaders(cls):
-        # init the SSL module
-        from pypy.module._ssl.interp_ssl import constants, HAVE_OPENSSL_RAND
-
-        for constant, value in constants.iteritems():
-            if constant.startswith('OP_'):
-                value = intmask(value)  # Convert to C long and wrap around.
-            Module.interpleveldefs[constant] = "space.wrap(%r)" % (value,)
-
-        if HAVE_OPENSSL_RAND:
-            Module.interpleveldefs['RAND_add'] = "interp_ssl.RAND_add"
-            Module.interpleveldefs['RAND_bytes'] = "interp_ssl.RAND_bytes"
-            Module.interpleveldefs['RAND_pseudo_bytes'] = "interp_ssl.RAND_pseudo_bytes"
-            Module.interpleveldefs['RAND_status'] = "interp_ssl.RAND_status"
-            Module.interpleveldefs['RAND_egd'] = "interp_ssl.RAND_egd"
-
-        for name, value in ssl_data.ALERT_DESCRIPTION_CODES.items():
-            Module.interpleveldefs[name] = "space.wrap(%r)" % value
-
-        super(Module, cls).buildloaders()
-
-    def setup_after_space_initialization(self):
-        """NOT_RPYTHON"""
-        from pypy.module._ssl.interp_ssl import PWINFO_STORAGE
-        PWINFO_STORAGE.clear()
-
-    def startup(self, space):
-        from rpython.rlib.ropenssl import init_ssl
-        init_ssl()
-        if space.config.objspace.usemodules.thread:
-            from pypy.module._ssl.thread_lock import setup_ssl_threads
-            setup_ssl_threads()
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
deleted file mode 100644
--- a/pypy/module/_ssl/interp_ssl.py
+++ /dev/null
@@ -1,1913 +0,0 @@
-import weakref
-
-from rpython.rlib import rpoll, rsocket, rthread, rweakref, rgc
-from rpython.rlib.rarithmetic import intmask, widen, r_uint
-from rpython.rlib.ropenssl import *
-from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
-from rpython.rlib.rposix import get_saved_errno
-from rpython.rlib.rweakref import RWeakValueDictionary
-from rpython.rlib.objectmodel import specialize, compute_unique_id
-from rpython.rtyper.lltypesystem import lltype, rffi
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.interpreter.unicodehelper import fsdecode
-from pypy.module._ssl.ssl_data import (
-    LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES)
-from pypy.module._socket import interp_socket
-from pypy.module.exceptions import interp_exceptions
-
-
-# user defined constants
-X509_NAME_MAXLEN = 256
-# these mirror ssl.h
-PY_SSL_ERROR_NONE, PY_SSL_ERROR_SSL = 0, 1
-PY_SSL_ERROR_WANT_READ, PY_SSL_ERROR_WANT_WRITE = 2, 3
-PY_SSL_ERROR_WANT_X509_LOOKUP = 4
-PY_SSL_ERROR_SYSCALL = 5  # look at error stack/return value/errno
-PY_SSL_ERROR_ZERO_RETURN, PY_SSL_ERROR_WANT_CONNECT = 6, 7
-# start of non ssl.h errorcodes
-PY_SSL_ERROR_EOF = 8  # special case of SSL_ERROR_SYSCALL
-PY_SSL_ERROR_INVALID_ERROR_CODE = 9
-
-PY_SSL_CERT_NONE, PY_SSL_CERT_OPTIONAL, PY_SSL_CERT_REQUIRED = 0, 1, 2
-
-PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1
-
-(PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3,
- PY_SSL_VERSION_TLS, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1,
- PY_SSL_VERSION_TLS1_2) = range(6)
-
-
-SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1
-SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3
-SOCKET_TOO_LARGE_FOR_SELECT, SOCKET_OPERATION_OK = 4, 5
-
-HAVE_RPOLL = 'poll' in dir(rpoll)
-
-constants = {}
-constants["SSL_ERROR_ZERO_RETURN"] = PY_SSL_ERROR_ZERO_RETURN
-constants["SSL_ERROR_WANT_READ"] = PY_SSL_ERROR_WANT_READ
-constants["SSL_ERROR_WANT_WRITE"] = PY_SSL_ERROR_WANT_WRITE
-constants["SSL_ERROR_WANT_X509_LOOKUP"] = PY_SSL_ERROR_WANT_X509_LOOKUP
-constants["SSL_ERROR_SYSCALL"] = PY_SSL_ERROR_SYSCALL
-constants["SSL_ERROR_SSL"] = PY_SSL_ERROR_SSL
-constants["SSL_ERROR_WANT_CONNECT"] = PY_SSL_ERROR_WANT_CONNECT
-constants["SSL_ERROR_EOF"] = PY_SSL_ERROR_EOF
-constants["SSL_ERROR_INVALID_ERROR_CODE"] = PY_SSL_ERROR_INVALID_ERROR_CODE
-
-constants["CERT_NONE"]     = PY_SSL_CERT_NONE
-constants["CERT_OPTIONAL"] = PY_SSL_CERT_OPTIONAL
-constants["CERT_REQUIRED"] = PY_SSL_CERT_REQUIRED
-
-constants["VERIFY_DEFAULT"] = 0
-constants["VERIFY_CRL_CHECK_LEAF"] = X509_V_FLAG_CRL_CHECK
-constants["VERIFY_CRL_CHECK_CHAIN"] = X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL
-constants["VERIFY_X509_STRICT"] = X509_V_FLAG_X509_STRICT
-
-constants["HAS_SNI"] = HAS_SNI
-constants["HAS_TLS_UNIQUE"] = HAVE_OPENSSL_FINISHED
-constants["HAS_ECDH"] = not OPENSSL_NO_ECDH
-constants["HAS_NPN"] = HAS_NPN
-constants["HAS_ALPN"] = HAS_ALPN
-
-constants["PROTOCOL_TLS"] = PY_SSL_VERSION_TLS
-constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_TLS  # Legacy name
-if not OPENSSL_NO_SSL2:
-    constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-if not OPENSSL_NO_SSL3:
-    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
-constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
-if HAVE_TLSv1_2:
-    constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1
-    constants["OP_NO_TLSv1_1"] = SSL_OP_NO_TLSv1_1
-    constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2
-    constants["OP_NO_TLSv1_2"] = SSL_OP_NO_TLSv1_2
-
-# protocol options
-constants["OP_ALL"] = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2
-constants["OP_NO_SSLv3"] = SSL_OP_NO_SSLv3
-constants["OP_NO_TLSv1"] = SSL_OP_NO_TLSv1
-constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE
-constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE
-constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE
-if SSL_OP_NO_COMPRESSION is not None:
-    constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION
-
-# OpenSSL version
-constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER
-ver = OPENSSL_VERSION_NUMBER
-ver, status = divmod(ver, 16)
-ver, patch  = divmod(ver, 256)
-ver, fix    = divmod(ver, 256)
-ver, minor  = divmod(ver, 256)
-ver, major  = divmod(ver, 256)
-version_info = (major, minor, fix, patch, status)
-constants["OPENSSL_VERSION_INFO"] = version_info
-constants["_OPENSSL_API_VERSION"] = version_info
-constants["OPENSSL_VERSION"] = SSLEAY_VERSION
-
-
-def ssl_error(space, msg, errno=0, w_errtype=None, errcode=0):
-    reason_str = None
-    lib_str = None
-    if errcode:
-        err_lib = libssl_ERR_GET_LIB(errcode)
-        err_reason = libssl_ERR_GET_REASON(errcode)
-        reason_str = ERROR_CODES_TO_NAMES.get((err_lib, err_reason), None)
-        lib_str = LIBRARY_CODES_TO_NAMES.get(err_lib, None)
-        raw_msg = libssl_ERR_reason_error_string(errcode)
-        msg = None
-        if raw_msg:
-            msg = rffi.charp2str(raw_msg)
-    if not msg:
-        msg = "unknown error"
-    if reason_str and lib_str:
-        msg = "[%s: %s] %s" % (lib_str, reason_str, msg)
-    elif lib_str:
-        msg = "[%s] %s" % (lib_str, msg)
-
-    w_exception_class = w_errtype or get_error(space).w_error
-    if errno or errcode:
-        w_exception = space.call_function(w_exception_class,
-                                          space.wrap(errno), space.wrap(msg))
-    else:
-        w_exception = space.call_function(w_exception_class, space.wrap(msg))
-    space.setattr(w_exception, space.wrap("reason"),
-                  space.wrap(reason_str) if reason_str else space.w_None)
-    space.setattr(w_exception, space.wrap("library"),
-                  space.wrap(lib_str) if lib_str else space.w_None)
-    return OperationError(w_exception_class, w_exception)
-
-def timeout_error(space, msg):
-    w_exc_class = interp_socket.get_error(space, 'timeout')
-    w_exc = space.call_function(w_exc_class, space.wrap(msg))
-    return OperationError(w_exc_class, w_exc)
-
-class SSLNpnProtocols(object):
-
-    def __init__(self, ctx, protos):
-        self.protos = protos
-        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
-        NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
-
-        # set both server and client callbacks, because the context
-        # can be used to create both types of sockets
-        libssl_SSL_CTX_set_next_protos_advertised_cb(
-            ctx, self.advertiseNPN_cb, self.buf)
-        libssl_SSL_CTX_set_next_proto_select_cb(
-            ctx, self.selectNPN_cb, self.buf)
-
-    def __del__(self):
-        rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.bufflag)
-
-    @staticmethod
-    def advertiseNPN_cb(s, data_ptr, len_ptr, args):
-        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if npn and npn.protos:
-            data_ptr[0] = npn.buf
-            len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
-        else:
-            data_ptr[0] = lltype.nullptr(rffi.CCHARP.TO)
-            len_ptr[0] = rffi.cast(rffi.UINT, 0)
-
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-    @staticmethod
-    def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
-        npn = NPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if npn and npn.protos:
-            client = npn.buf
-            client_len = len(npn.protos)
-        else:
-            client = lltype.nullptr(rffi.CCHARP.TO)
-            client_len = 0
-
-        libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
-                                     server, server_len,
-                                     client, client_len)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-
-class SSLAlpnProtocols(object):
-
-    def __init__(self, ctx, protos):
-        self.protos = protos
-        self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
-        ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
-
-        with rffi.scoped_str2charp(protos) as protos_buf:
-            if libssl_SSL_CTX_set_alpn_protos(
-                    ctx, rffi.cast(rffi.UCHARP, protos_buf), len(protos)):
-                raise MemoryError
-        libssl_SSL_CTX_set_alpn_select_cb(
-            ctx, self.selectALPN_cb, self.buf)
-
-    def __del__(self):
-        rffi.free_nonmovingbuffer(
-            self.protos, self.buf, self.bufflag)
-
-    @staticmethod
-    def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
-        alpn = ALPN_STORAGE.get(rffi.cast(lltype.Unsigned, args))
-        if alpn and alpn.protos:
-            server = alpn.buf
-            server_len = len(alpn.protos)
-        else:
-            server = lltype.nullptr(rffi.CCHARP.TO)
-            server_len = 0
-
-        ret = libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
-                                           server, server_len,
-                                           client, client_len)
-        if ret != OPENSSL_NPN_NEGOTIATED:
-            return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_NOACK)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-
-
-NPN_STORAGE = RWeakValueDictionary(r_uint, SSLNpnProtocols)
-ALPN_STORAGE = RWeakValueDictionary(r_uint, SSLAlpnProtocols)
-
-SOCKET_STORAGE = RWeakValueDictionary(int, W_Root)
-
-
-if HAVE_OPENSSL_RAND:
-    # helper routines for seeding the SSL PRNG
-    @unwrap_spec(string=str, entropy=float)
-    def RAND_add(space, string, entropy):
-        """RAND_add(string, entropy)
-
-
-        Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
-        bound on the entropy contained in string."""
-        with rffi.scoped_nonmovingbuffer(string) as buf:
-            libssl_RAND_add(buf, len(string), entropy)
-
-    def _RAND_bytes(space, n, pseudo):
-        if n < 0:
-            raise oefmt(space.w_ValueError, "num must be positive")
-
-        with rffi.scoped_alloc_buffer(n) as buf:
-            if pseudo:
-                ok = libssl_RAND_pseudo_bytes(
-                    rffi.cast(rffi.UCHARP, buf.raw), n)
-                if ok == 0 or ok == 1:
-                    return space.newtuple([
-                        space.newbytes(buf.str(n)),
-                        space.wrap(ok == 1),
-                    ])
-            else:
-                ok = libssl_RAND_bytes(
-                    rffi.cast(rffi.UCHARP, buf.raw), n)
-                if ok == 1:
-                    return space.newbytes(buf.str(n))
-
-        raise ssl_error(space, "", errcode=libssl_ERR_get_error())
-
-    @unwrap_spec(n=int)
-    def RAND_bytes(space, n):
-        """RAND_bytes(n) -> bytes
-
-        Generate n cryptographically strong pseudo-random bytes."""
-        return _RAND_bytes(space, n, pseudo=False)
-
-    @unwrap_spec(n=int)
-    def RAND_pseudo_bytes(space, n):
-        """RAND_pseudo_bytes(n) -> (bytes, is_cryptographic)
-
-        Generate n pseudo-random bytes. is_cryptographic is True if the bytes
-        generated are cryptographically strong."""
-        return _RAND_bytes(space, n, pseudo=True)
-
-    def RAND_status(space):
-        """RAND_status() -> 0 or 1
-
-        Returns 1 if the OpenSSL PRNG has been seeded with enough data
-        and 0 if not.  It is necessary to seed the PRNG with RAND_add()
-        on some platforms before using the ssl() function."""
-
-        res = libssl_RAND_status()
-        return space.wrap(res)
-
-    if HAVE_OPENSSL_RAND_EGD:
-        @unwrap_spec(path=str)
-        def RAND_egd(space, path):
-            """RAND_egd(path) -> bytes
-
-            Queries the entropy gather daemon (EGD) on socket path.  Returns number
-            of bytes read.  Raises socket.sslerror if connection to EGD fails or
-            if it does provide enough data to seed PRNG."""
-            with rffi.scoped_str2charp(path) as socket_path:
-                bytes = libssl_RAND_egd(socket_path)
-            if bytes == -1:
-                raise ssl_error(space,
-                                "EGD connection failed or EGD did not return "
-                                "enough data to seed the PRNG")
-            return space.wrap(bytes)
-    else:
-        # Dummy func for platforms missing RAND_egd(). Most likely LibreSSL.
-        @unwrap_spec(path=str)
-        def RAND_egd(space, path):
-            raise ssl_error(space, "RAND_egd unavailable")
-
-
-class SSLSocket(W_Root):
-    def __init__(self, space, w_ctx):
-        self.w_ctx = w_ctx
-        self.w_socket = None
-        self.w_owner = None
-        self.ssl = lltype.nullptr(SSL.TO)
-        self.peer_cert = lltype.nullptr(X509.TO)
-        self.shutdown_seen_zero = False
-        self.handshake_done = False
-        self.register_finalizer(space)
-
-    def _finalize_(self):
-        peer_cert = self.peer_cert
-        if peer_cert:
-            self.peer_cert = lltype.nullptr(X509.TO)
-            libssl_X509_free(peer_cert)
-        ssl = self.ssl
-        if ssl:
-            self.ssl = lltype.nullptr(SSL.TO)
-            libssl_SSL_free(ssl)
-
-    @unwrap_spec(data='bufferstr')
-    def write(self, space, data):
-        """write(s) -> len
-
-        Writes the string s into the SSL object.  Returns the number
-        of bytes written."""
-        w_socket = self._get_socket(space)
-
-        sockstate = checkwait(space, w_socket, True)
-        if sockstate == SOCKET_HAS_TIMED_OUT:
-            raise timeout_error(space, "The write operation timed out")
-        elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-            raise ssl_error(space, "Underlying socket has been closed.")
-        elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-            raise ssl_error(space, "Underlying socket too large for select().")
-
-        num_bytes = 0
-        while True:
-            err = 0
-
-            num_bytes = libssl_SSL_write(self.ssl, data, len(data))
-            err = libssl_SSL_get_error(self.ssl, num_bytes)
-
-            if err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, w_socket, False)
-            elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, w_socket, True)
-            else:
-                sockstate = SOCKET_OPERATION_OK
-
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise timeout_error(space, "The write operation timed out")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                raise ssl_error(space, "Underlying socket has been closed.")
-            elif sockstate == SOCKET_IS_NONBLOCKING:
-                break
-
-            if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                continue
-            else:
-                break
-
-        if num_bytes > 0:
-            return space.wrap(num_bytes)
-        else:
-            raise _ssl_seterror(space, self, num_bytes)
-
-    def pending(self, space):
-        """pending() -> count
-
-        Returns the number of already decrypted bytes available for read,
-        pending on the connection."""
-        count = libssl_SSL_pending(self.ssl)
-        if count < 0:
-            raise _ssl_seterror(space, self, count)
-        return space.wrap(count)
-
-    @unwrap_spec(num_bytes=int)
-    def read(self, space, num_bytes, w_buffer=None):
-        """read([len]) -> string
-
-        Read up to len bytes from the SSL socket."""
-        w_socket = self._get_socket(space)
-
-        count = libssl_SSL_pending(self.ssl)
-        if not count:
-            sockstate = checkwait(space, w_socket, False)
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise timeout_error(space, "The read operation timed out")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN:
-                    if space.is_none(w_buffer):
-                        return space.newbytes('')
-                    else:
-                        return space.wrap(0)
-                raise ssl_error(space,
-                                "Socket closed without SSL shutdown handshake")
-
-        if w_buffer:
-            rwbuffer = space.getarg_w('w*', w_buffer)
-            buflen = rwbuffer.getlength()
-            if not 0 < num_bytes <= buflen:
-                num_bytes = buflen
-        else:
-            if num_bytes < 0:
-                raise oefmt(space.w_ValueError, "size should not be negative")
-            rwbuffer = None
-
-        with rffi.scoped_alloc_buffer(num_bytes) as buf:
-            while True:
-                err = 0
-
-                count = libssl_SSL_read(self.ssl, buf.raw, num_bytes)
-                err = libssl_SSL_get_error(self.ssl, count)
-
-                if err == SSL_ERROR_WANT_READ:
-                    sockstate = checkwait(space, w_socket, False)
-                elif err == SSL_ERROR_WANT_WRITE:
-                    sockstate = checkwait(space, w_socket, True)
-                elif (err == SSL_ERROR_ZERO_RETURN and
-                      libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
-                    if space.is_none(w_buffer):
-                        return space.newbytes('')
-                    else:
-                        return space.wrap(0)
-                else:
-                    sockstate = SOCKET_OPERATION_OK
-
-                if sockstate == SOCKET_HAS_TIMED_OUT:
-                    raise timeout_error(space, "The read operation timed out")
-                elif sockstate == SOCKET_IS_NONBLOCKING:
-                    break
-
-                if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                    continue
-                else:
-                    break
-
-            if count <= 0:
-                raise _ssl_seterror(space, self, count)
-
-            result = buf.str(count)
-
-        if rwbuffer is not None:
-            rwbuffer.setslice(0, result)
-            return space.wrap(count)
-        else:
-            return space.newbytes(result)
-
-    def _get_socket(self, space):
-        w_socket = self.w_socket()
-        if w_socket is None:
-            raise ssl_error(space, "Underlying socket connection gone")
-
-        # just in case the blocking state of the socket has been changed
-        w_timeout = space.call_method(w_socket, "gettimeout")
-        nonblocking = not space.is_w(w_timeout, space.w_None)
-        libssl_BIO_set_nbio(libssl_SSL_get_rbio(self.ssl), nonblocking)
-        libssl_BIO_set_nbio(libssl_SSL_get_wbio(self.ssl), nonblocking)
-
-        return w_socket
-
-    def do_handshake(self, space):
-        w_socket = self._get_socket(space)
-
-        # Actually negotiate SSL connection
-        # XXX If SSL_do_handshake() returns 0, it's also a failure.
-        while True:
-            ret = libssl_SSL_do_handshake(self.ssl)
-            err = libssl_SSL_get_error(self.ssl, ret)
-            # XXX PyErr_CheckSignals()
-            if err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, w_socket, False)
-            elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, w_socket, True)
-            else:
-                sockstate = SOCKET_OPERATION_OK
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                raise timeout_error(space, "The handshake operation timed out")
-            elif sockstate == SOCKET_HAS_BEEN_CLOSED:
-                raise ssl_error(space, "Underlying socket has been closed.")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate == SOCKET_IS_NONBLOCKING:
-                break
-
-            if err == SSL_ERROR_WANT_READ or err == SSL_ERROR_WANT_WRITE:
-                continue
-            else:
-                break
-
-        if ret <= 0:
-            raise _ssl_seterror(space, self, ret)
-
-        if self.peer_cert:
-            libssl_X509_free(self.peer_cert)
-        self.peer_cert = libssl_SSL_get_peer_certificate(self.ssl)
-        self.handshake_done = True
-
-    def shutdown(self, space):
-        w_socket = self._get_socket(space)
-
-        # Guard against closed socket
-        w_fileno = space.call_method(w_socket, "fileno")
-        if space.int_w(w_fileno) < 0:
-            raise ssl_error(space, "Underlying socket has been closed")
-
-        zeros = 0
-        while True:
-            # Disable read-ahead so that unwrap can work correctly.
-            # Otherwise OpenSSL might read in too much data,
-            # eating clear text data that happens to be
-            # transmitted after the SSL shutdown.
-            # Should be safe to call repeatedly everytime this
-            # function is used and the shutdown_seen_zero != 0
-            # condition is met.
-            if self.shutdown_seen_zero:
-                libssl_SSL_set_read_ahead(self.ssl, 0)
-            ret = libssl_SSL_shutdown(self.ssl)
-
-            # if err == 1, a secure shutdown with SSL_shutdown() is complete
-            if ret > 0:
-                break
-            if ret == 0:
-                # Don't loop endlessly; instead preserve legacy
-                # behaviour of trying SSL_shutdown() only twice.
-                # This looks necessary for OpenSSL < 0.9.8m
-                zeros += 1
-                if zeros > 1:
-                    break
-                # Shutdown was sent, now try receiving
-                self.shutdown_seen_zero = True
-                continue
-
-            # Possibly retry shutdown until timeout or failure
-            ssl_err = libssl_SSL_get_error(self.ssl, ret)
-            if ssl_err == SSL_ERROR_WANT_READ:
-                sockstate = checkwait(space, w_socket, False)
-            elif ssl_err == SSL_ERROR_WANT_WRITE:
-                sockstate = checkwait(space, w_socket, True)
-            else:
-                break
-
-            if sockstate == SOCKET_HAS_TIMED_OUT:
-                if ssl_err == SSL_ERROR_WANT_READ:
-                    raise timeout_error(space, "The read operation timed out")
-                else:
-                    raise timeout_error(space, "The write operation timed out")
-            elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space,
-                                "Underlying socket too large for select().")
-            elif sockstate != SOCKET_OPERATION_OK:
-                # Retain the SSL error code
-                break
-
-        if ret < 0:
-            raise _ssl_seterror(space, self, ret)
-
-        return w_socket
-
-    def cipher(self, space):
-        if not self.ssl:
-            return space.w_None
-        current = libssl_SSL_get_current_cipher(self.ssl)
-        if not current:
-            return space.w_None
-
-        name = libssl_SSL_CIPHER_get_name(current)
-        w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None
-
-        proto = libssl_SSL_CIPHER_get_version(current)
-        w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None
-
-        bits = libssl_SSL_CIPHER_get_bits(current,
-                                          lltype.nullptr(rffi.INTP.TO))
-        w_bits = space.newint(bits)
-        return space.newtuple([w_name, w_proto, w_bits])
-
-    @unwrap_spec(der=bool)
-    def peer_certificate(self, space, der=False):
-        """peer_certificate([der=False]) -> certificate
-
-        Returns the certificate for the peer.  If no certificate was
-        provided, returns None.  If a certificate was provided, but not
-        validated, returns an empty dictionary.  Otherwise returns a
-        dict containing information about the peer certificate.
-
-        If the optional argument is True, returns a DER-encoded copy of
-        the peer certificate, or None if no certificate was provided.
-        This will return the certificate even if it wasn't validated.
-        """
-        if not self.handshake_done:
-            raise oefmt(space.w_ValueError, "hanshake not done yet")
-        if not self.peer_cert:
-            return space.w_None
-
-        if der:
-            # return cert in DER-encoded format
-            return _certificate_to_der(space, self.peer_cert)
-        else:
-            verification = libssl_SSL_CTX_get_verify_mode(
-                libssl_SSL_get_SSL_CTX(self.ssl))
-            if not verification & SSL_VERIFY_PEER:
-                return space.newdict()
-            else:
-                return _decode_certificate(space, self.peer_cert)
-
-    def selected_npn_protocol(self, space):
-        if not HAS_NPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The NPN extension requires OpenSSL 1.0.1 or later.")
-        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
-            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
-                libssl_SSL_get0_next_proto_negotiated(self.ssl,
-                                                      out_ptr, len_ptr)
-                if out_ptr[0]:
-                    return space.wrap(
-                        rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
-
-    def selected_alpn_protocol(self, space):
-        if not HAS_ALPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
-        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
-            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
-                libssl_SSL_get0_alpn_selected(self.ssl,
-                                              out_ptr, len_ptr)
-                if out_ptr[0]:
-                    return space.wrap(
-                        rffi.charpsize2str(out_ptr[0], intmask(len_ptr[0])))
-
-    def compression_w(self, space):
-        if not self.ssl:
-            return space.w_None
-        comp_method = libssl_SSL_get_current_compression(self.ssl)
-        if not comp_method:
-            return space.w_None
-        method_type = intmask(libssl_COMP_get_type(comp_method))
-        if method_type == NID_undef:
-            return space.w_None
-        short_name = libssl_COMP_get_name(comp_method)
-        if not short_name:
-            return space.w_None
-        return space.wrap(rffi.charp2str(short_name))
-
-    def version_w(self, space):
-        if not self.ssl:
-            return space.w_None
-        version = libssl_SSL_get_version(self.ssl)
-        if not version:
-            return space.w_None
-        return space.wrap(rffi.charp2str(version))
-
-    def tls_unique_cb_w(self, space):
-        """Returns the 'tls-unique' channel binding data, as defined by RFC 5929.
-        If the TLS handshake is not yet complete, None is returned"""
-
-        # In case of 'tls-unique' it will be 12 bytes for TLS, 36
-        # bytes for older SSL, but let's be safe
-        CB_MAXLEN = 128
-
-        with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf:
-            if (libssl_SSL_session_reused(self.ssl) ^
-                (self.socket_type == PY_SSL_CLIENT)):
-                # if session is resumed XOR we are the client
-                length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN)
-            else:
-                # if a new session XOR we are the server
-                length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN)
-
-            if length > 0:
-                return space.newbytes(rffi.charpsize2str(buf, intmask(length)))
-
-    def descr_get_context(self, space):
-        return self.w_ctx
-
-    def descr_set_context(self, space, w_ctx):
-        ctx = space.interp_w(SSLContext, w_ctx)
-        if not HAS_SNI:
-            raise oefmt(space.w_NotImplementedError,
-                        "setting a socket's context "
-                        "is not supported by your OpenSSL library")
-        self.w_ctx = w_ctx
-        libssl_SSL_set_SSL_CTX(self.ssl, ctx.ctx)
-
-    def descr_get_owner(self, space):
-        if self.w_owner is not None:
-            w_owner = self.w_owner()
-            if w_owner:
-                return w_owner
-        return space.w_None
-
-    def descr_set_owner(self, space, w_owner):
-        assert w_owner is not None
-        self.w_owner = weakref.ref(w_owner)
-
-
-SSLSocket.typedef = TypeDef("_ssl._SSLSocket",
-    write = interp2app(SSLSocket.write),
-    pending = interp2app(SSLSocket.pending),
-    read = interp2app(SSLSocket.read),
-    do_handshake = interp2app(SSLSocket.do_handshake),
-    shutdown = interp2app(SSLSocket.shutdown),
-    cipher = interp2app(SSLSocket.cipher),
-    peer_certificate = interp2app(SSLSocket.peer_certificate),
-    selected_npn_protocol = interp2app(SSLSocket.selected_npn_protocol),
-    selected_alpn_protocol = interp2app(SSLSocket.selected_alpn_protocol),
-    compression = interp2app(SSLSocket.compression_w),
-    version = interp2app(SSLSocket.version_w),
-    tls_unique_cb = interp2app(SSLSocket.tls_unique_cb_w),
-    context=GetSetProperty(SSLSocket.descr_get_context,
-                           SSLSocket.descr_set_context),
-    owner=GetSetProperty(SSLSocket.descr_get_owner,
-                           SSLSocket.descr_set_owner),
-)
-
-def _certificate_to_der(space, certificate):
-    with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as buf_ptr:
-        buf_ptr[0] = lltype.nullptr(rffi.CCHARP.TO)
-        length = libssl_i2d_X509(certificate, buf_ptr)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            return space.newbytes(rffi.charpsize2str(buf_ptr[0], length))
-        finally:
-            libssl_OPENSSL_free(buf_ptr[0])
-
-def _decode_certificate(space, certificate):
-    w_retval = space.newdict()
-
-    w_peer = _create_tuple_for_X509_NAME(
-        space, libssl_X509_get_subject_name(certificate))
-    space.setitem(w_retval, space.wrap("subject"), w_peer)
-
-    w_issuer = _create_tuple_for_X509_NAME(
-        space, libssl_X509_get_issuer_name(certificate))
-    space.setitem(w_retval, space.wrap("issuer"), w_issuer)
-
-    space.setitem(w_retval, space.wrap("version"),
-                  space.wrap(libssl_X509_get_version(certificate) + 1))
-
-    biobuf = libssl_BIO_new(libssl_BIO_s_mem())
-    try:
-
-        libssl_BIO_reset(biobuf)
-        serialNumber = libssl_X509_get_serialNumber(certificate)
-        libssl_i2a_ASN1_INTEGER(biobuf, serialNumber)
-        # should not exceed 20 octets, 160 bits, so buf is big enough
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-
-            w_serial = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("serialNumber"), w_serial)
-
-        libssl_BIO_reset(biobuf)
-        notBefore = libssl_X509_get_notBefore(certificate)
-        libssl_ASN1_TIME_print(biobuf, notBefore)
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-            w_date = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("notBefore"), w_date)
-
-        libssl_BIO_reset(biobuf)
-        notAfter = libssl_X509_get_notAfter(certificate)
-        libssl_ASN1_TIME_print(biobuf, notAfter)
-        with lltype.scoped_alloc(rffi.CCHARP.TO, 100) as buf:
-            length = libssl_BIO_gets(biobuf, buf, 99)
-            if length < 0:
-                raise _ssl_seterror(space, None, length)
-            w_date = space.wrap(rffi.charpsize2str(buf, length))
-        space.setitem(w_retval, space.wrap("notAfter"), w_date)
-    finally:
-        libssl_BIO_free(biobuf)
-
-    # Now look for subjectAltName
-    w_alt_names = _get_peer_alt_names(space, certificate)
-    if w_alt_names is not space.w_None:
-        space.setitem(w_retval, space.wrap("subjectAltName"), w_alt_names)
-
-    # Authority Information Access: OCSP URIs
-    w_ocsp = _get_aia_uri(space, certificate, NID_ad_OCSP)
-    if not space.is_none(w_ocsp):
-        space.setitem(w_retval, space.wrap("OCSP"), w_ocsp)
-    w_issuers = _get_aia_uri(space, certificate, NID_ad_ca_issuers)
-    if not space.is_none(w_issuers):
-        space.setitem(w_retval, space.wrap("caIssuers"), w_issuers)
-
-    # CDP (CRL distribution points)
-    w_cdp = _get_crl_dp(space, certificate)
-    if not space.is_none(w_cdp):
-        space.setitem(w_retval, space.wrap("crlDistributionPoints"), w_cdp)
-
-    return w_retval
-
-
-def _create_tuple_for_X509_NAME(space, xname):
-    entry_count = libssl_X509_NAME_entry_count(xname)
-    dn_w = []
-    rdn_w = []
-    rdn_level = -1
-    for index in range(entry_count):
-        entry = libssl_X509_NAME_get_entry(xname, index)
-        # check to see if we've gotten to a new RDN
-        entry_level = intmask(libssl_X509_NAME_ENTRY_set(entry))
-        if rdn_level >= 0:
-            if rdn_level != entry_level:
-                # yes, new RDN
-                # add old RDN to DN
-                dn_w.append(space.newtuple(list(rdn_w)))
-                rdn_w = []
-        rdn_level = entry_level
-
-        # Now add this attribute to the current RDN
-        name = libssl_X509_NAME_ENTRY_get_object(entry)
-        value = libssl_X509_NAME_ENTRY_get_data(entry)
-        attr = _create_tuple_for_attribute(space, name, value)
-        rdn_w.append(attr)
-
-    # Now, there is typically a dangling RDN
-    if rdn_w:
-        dn_w.append(space.newtuple(list(rdn_w)))
-    return space.newtuple(list(dn_w))
-
-
-def _get_peer_alt_names(space, certificate):
-    # this code follows the procedure outlined in
-    # OpenSSL's crypto/x509v3/v3_prn.c:X509v3_EXT_print()
-    # function to extract the STACK_OF(GENERAL_NAME),
-    # then iterates through the stack to add the
-    # names.
-
-    if not certificate:
-        return space.w_None
-
-    # get a memory buffer
-    biobuf = libssl_BIO_new(libssl_BIO_s_mem())
-
-    try:
-        alt_names_w = []
-        i = -1
-        while True:
-            i = libssl_X509_get_ext_by_NID(
-                certificate, NID_subject_alt_name, i)
-            if i < 0:
-                break
-
-            # now decode the altName
-            ext = libssl_X509_get_ext(certificate, i)
-            method = libssl_X509V3_EXT_get(ext)
-            if not method:
-                raise ssl_error(space,
-                                "No method for internalizing subjectAltName!'")
-
-            with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as p_ptr:
-                ext_value = libssl_X509_EXTENSION_get_data(ext)
-                p_ptr[0] = ext_value.c_data
-                length = intmask(ext_value.c_length)
-                null = lltype.nullptr(rffi.VOIDP.TO)
-                if method[0].c_it:
-                    names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
-                        null, p_ptr, length,
-                        libssl_ASN1_ITEM_ptr(method[0].c_it)))
-                else:
-                    names = rffi.cast(GENERAL_NAMES, method[0].c_d2i(
-                        null, p_ptr, length))
-
-            try:
-                for j in range(libssl_sk_GENERAL_NAME_num(names)):
-                    # Get a rendering of each name in the set of names
-
-                    name = libssl_sk_GENERAL_NAME_value(names, j)
-                    gntype = intmask(name.c_type)
-                    if gntype == GEN_DIRNAME:
-                        # we special-case DirName as a tuple of tuples of
-                        # attributes
-                        dirname = libssl_pypy_GENERAL_NAME_dirn(name)
-                        w_t = space.newtuple([
-                            space.wrap("DirName"),
-                            _create_tuple_for_X509_NAME(space, dirname)
-                            ])
-                    elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
-                        # GENERAL_NAME_print() doesn't handle NULL bytes in
-                        # ASN1_string correctly, CVE-2013-4238
-                        if gntype == GEN_EMAIL:
-                            v = space.wrap("email")
-                        elif gntype == GEN_DNS:
-                            v = space.wrap("DNS")
-                        elif gntype == GEN_URI:
-                            v = space.wrap("URI")
-                        else:
-                            assert False
-                        as_ = libssl_pypy_GENERAL_NAME_dirn(name)
-                        as_ = rffi.cast(ASN1_STRING, as_)
-                        buf = libssl_ASN1_STRING_data(as_)
-                        length = libssl_ASN1_STRING_length(as_)
-                        w_t = space.newtuple([
-                            v, space.wrap(rffi.charpsize2str(buf, length))])
-                    else:
-                        # for everything else, we use the OpenSSL print form
-                        if gntype not in (GEN_OTHERNAME, GEN_X400, GEN_EDIPARTY,
-                                          GEN_IPADD, GEN_RID):
-                            space.warn(space.wrap("Unknown general name type"),
-                                       space.w_RuntimeWarning)
-                        libssl_BIO_reset(biobuf)
-                        libssl_GENERAL_NAME_print(biobuf, name)
-                        with lltype.scoped_alloc(rffi.CCHARP.TO, 2048) as buf:
-                            length = libssl_BIO_gets(biobuf, buf, 2047)
-                            if length < 0:
-                                raise _ssl_seterror(space, None, 0)
-
-                            v = rffi.charpsize2str(buf, length)
-                        v1, v2 = v.split(':', 1)
-                        w_t = space.newtuple([space.wrap(v1),
-                                              space.wrap(v2)])
-
-                    alt_names_w.append(w_t)
-            finally:
-                libssl_pypy_GENERAL_NAME_pop_free(names)
-    finally:
-        libssl_BIO_free(biobuf)
-
-    if alt_names_w:
-        return space.newtuple(list(alt_names_w))
-    else:
-        return space.w_None
-
-
-def _create_tuple_for_attribute(space, name, value):
-    with lltype.scoped_alloc(rffi.CCHARP.TO, X509_NAME_MAXLEN) as buf:
-        length = libssl_OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        w_name = space.wrap(rffi.charpsize2str(buf, length))
-
-    with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as buf_ptr:
-        length = libssl_ASN1_STRING_to_UTF8(buf_ptr, value)
-        if length < 0:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            w_value = space.newbytes(rffi.charpsize2str(buf_ptr[0], length))
-            w_value = space.call_method(w_value, "decode", space.wrap("utf-8"))
-        finally:
-            libssl_OPENSSL_free(buf_ptr[0])
-
-    return space.newtuple([w_name, w_value])
-
-
-def _get_aia_uri(space, certificate, nid):
-    info = rffi.cast(AUTHORITY_INFO_ACCESS, libssl_X509_get_ext_d2i(
-        certificate, NID_info_access, None, None))
-    try:
-        if not info or libssl_sk_ACCESS_DESCRIPTION_num(info) == 0:
-            return
-
-        result_w = []
-        for i in range(libssl_sk_ACCESS_DESCRIPTION_num(info)):
-            ad = libssl_sk_ACCESS_DESCRIPTION_value(info, i)
-            if libssl_OBJ_obj2nid(ad[0].c_method) != nid:
-                continue
-
-            name = ad[0].c_location
-            gntype = intmask(name.c_type)
-            if gntype != GEN_URI:
-                continue
-            uri = libssl_pypy_GENERAL_NAME_uri(name)
-            length = intmask(uri.c_length)
-            s_uri = rffi.charpsize2str(uri.c_data, length)
-            result_w.append(space.wrap(s_uri))
-        return space.newtuple(result_w[:])
-    finally:
-        libssl_AUTHORITY_INFO_ACCESS_free(info)
-
-def _get_crl_dp(space, certificate):
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        # Calls x509v3_cache_extensions and sets up crldp
-        libssl_X509_check_ca(certificate)
-    dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
-        certificate, NID_crl_distribution_points, None, None))
-    if not dps:
-        return None
-
-    try:
-        cdp_w = []
-        for i in range(libssl_sk_DIST_POINT_num(dps)):
-            dp = libssl_sk_DIST_POINT_value(dps, i)
-            gns = libssl_pypy_DIST_POINT_fullname(dp)
-
-            for j in range(libssl_sk_GENERAL_NAME_num(gns)):
-                name = libssl_sk_GENERAL_NAME_value(gns, j)
-                gntype = intmask(name.c_type)
-                if gntype != GEN_URI:
-                    continue
-                uri = libssl_pypy_GENERAL_NAME_uri(name)
-                length = intmask(uri.c_length)
-                s_uri = rffi.charpsize2str(uri.c_data, length)
-                cdp_w.append(space.wrap(s_uri))
-    finally:
-        if OPENSSL_VERSION_NUMBER < 0x10001000:
-            libssl_sk_DIST_POINT_free(dps)
-    return space.newtuple(cdp_w[:])
-
-def new_sslobject(space, w_ctx, w_sock, side, server_hostname):
-    ss = SSLSocket(space, w_ctx)
-
-    sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-    w_timeout = space.call_method(w_sock, "gettimeout")
-    has_timeout = not space.is_none(w_timeout)
-
-    ss.ssl = libssl_SSL_new(w_ctx.ctx) # new ssl struct
-    libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL
-    # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
-    # of a str object may be changed by the garbage collector.
-    libssl_SSL_set_mode(
-        ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
-
-    if server_hostname:
-        libssl_SSL_set_tlsext_host_name(ss.ssl, server_hostname);
-
-    # If the socket is in non-blocking mode or timeout mode, set the BIO
-    # to non-blocking mode (blocking is the default)
-    if has_timeout:
-        # Set both the read and write BIO's to non-blocking mode
-        libssl_BIO_set_nbio(libssl_SSL_get_rbio(ss.ssl), 1)
-        libssl_BIO_set_nbio(libssl_SSL_get_wbio(ss.ssl), 1)
-
-    if side == PY_SSL_CLIENT:
-        libssl_SSL_set_connect_state(ss.ssl)
-    else:
-        libssl_SSL_set_accept_state(ss.ssl)
-    ss.socket_type = side
-
-    ss.w_socket = weakref.ref(w_sock)
-    return ss
-
-def checkwait(space, w_sock, writing):
-    """If the socket has a timeout, do a select()/poll() on the socket.
-    The argument writing indicates the direction.
-    Returns one of the possibilities in the timeout_state enum (above)."""
-
-    w_timeout = space.call_method(w_sock, "gettimeout")
-    if space.is_w(w_timeout, space.w_None):
-        return SOCKET_IS_BLOCKING
-    elif space.float_w(w_timeout) == 0.0:
-        return SOCKET_IS_NONBLOCKING
-    sock_timeout = space.float_w(w_timeout)
-
-    sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-
-    # guard against closed socket
-    if sock_fd < 0:
-        return SOCKET_HAS_BEEN_CLOSED
-
-    # see if the socket is ready
-
-    # Prefer poll, if available, since you can poll() any fd
-    # which can't be done with select().
-    if HAVE_RPOLL:
-        if writing:
-            fddict = {sock_fd: rpoll.POLLOUT}
-        else:
-            fddict = {sock_fd: rpoll.POLLIN}
-
-        # socket's timeout is in seconds, poll's timeout in ms
-        timeout = int(sock_timeout * 1000 + 0.5)
-        try:
-            ready = rpoll.poll(fddict, timeout)
-        except rpoll.PollError as e:
-            message = e.get_msg()
-            raise ssl_error(space, message, e.errno)
-    else:
-        if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
-            return SOCKET_TOO_LARGE_FOR_SELECT
-
-        try:
-            if writing:
-                r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
-                ready = w
-            else:
-                r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
-                ready = r
-        except rpoll.SelectError as e:
-            message = e.get_msg()
-            raise ssl_error(space, message, e.errno)
-    if ready:
-        return SOCKET_OPERATION_OK
-    else:
-        return SOCKET_HAS_TIMED_OUT
-
-
-def _ssl_seterror(space, ss, ret):
-    assert ret <= 0
-
-    errcode = libssl_ERR_peek_last_error()
-
-    if ss is None:
-        return ssl_error(space, None, errcode=errcode)
-    elif ss.ssl:
-        err = libssl_SSL_get_error(ss.ssl, ret)
-    else:
-        err = SSL_ERROR_SSL
-    w_errtype = None
-    errstr = ""
-    errval = 0
-
-    if err == SSL_ERROR_ZERO_RETURN:
-        w_errtype = get_error(space).w_ZeroReturnError
-        errstr = "TLS/SSL connection has been closed"
-        errval = PY_SSL_ERROR_ZERO_RETURN
-    elif err == SSL_ERROR_WANT_READ:
-        w_errtype = get_error(space).w_WantReadError
-        errstr = "The operation did not complete (read)"
-        errval = PY_SSL_ERROR_WANT_READ
-    elif err == SSL_ERROR_WANT_WRITE:
-        w_errtype = get_error(space).w_WantWriteError
-        errstr = "The operation did not complete (write)"
-        errval = PY_SSL_ERROR_WANT_WRITE
-    elif err == SSL_ERROR_WANT_X509_LOOKUP:
-        errstr = "The operation did not complete (X509 lookup)"
-        errval = PY_SSL_ERROR_WANT_X509_LOOKUP
-    elif err == SSL_ERROR_WANT_CONNECT:
-        errstr = "The operation did not complete (connect)"
-        errval = PY_SSL_ERROR_WANT_CONNECT
-    elif err == SSL_ERROR_SYSCALL:
-        e = libssl_ERR_get_error()
-        if e == 0:
-            if ret == 0 or ss.w_socket() is None:
-                w_errtype = get_error(space).w_EOFError
-                errstr = "EOF occurred in violation of protocol"
-                errval = PY_SSL_ERROR_EOF
-            elif ret == -1:
-                # the underlying BIO reported an I/0 error
-                error = rsocket.last_error()
-                return interp_socket.converted_error(space, error)
-            else:
-                w_errtype = get_error(space).w_SyscallError
-                errstr = "Some I/O error occurred"
-                errval = PY_SSL_ERROR_SYSCALL
-        else:
-            errstr = rffi.charp2str(libssl_ERR_error_string(e, None))
-            errval = PY_SSL_ERROR_SYSCALL
-    elif err == SSL_ERROR_SSL:
-        errval = PY_SSL_ERROR_SSL
-        if errcode != 0:
-            errstr = rffi.charp2str(libssl_ERR_error_string(errcode, None))
-        else:
-            errstr = "A failure in the SSL library occurred"
-    else:
-        errstr = "Invalid error code"
-        errval = PY_SSL_ERROR_INVALID_ERROR_CODE
-
-    return ssl_error(space, errstr, errval, w_errtype=w_errtype,
-                     errcode=errcode)
-
-def SSLError_descr_str(space, w_exc):
-    w_strerror = space.getattr(w_exc, space.wrap("strerror"))
-    if not space.is_none(w_strerror):
-        return w_strerror
-    return space.str(space.getattr(w_exc, space.wrap("args")))
-
-
-class ErrorCache:
-    def __init__(self, space):
-        w_socketerror = interp_socket.get_error(space, "error")
-        self.w_error = space.new_exception_class(
-            "_ssl.SSLError", w_socketerror)
-        space.setattr(self.w_error, space.wrap('__str__'),
-                      space.wrap(interp2app(SSLError_descr_str)))
-        self.w_ZeroReturnError = space.new_exception_class(
-            "ssl.SSLZeroReturnError", self.w_error)
-        self.w_WantReadError = space.new_exception_class(
-            "ssl.SSLWantReadError", self.w_error)
-        self.w_WantWriteError = space.new_exception_class(
-            "ssl.SSLWantWriteError", self.w_error)
-        self.w_EOFError = space.new_exception_class(
-            "ssl.SSLEOFError", self.w_error)
-        self.w_SyscallError = space.new_exception_class(
-            "ssl.SSLSyscallError", self.w_error)
-
-def get_error(space):
-    return space.fromcache(ErrorCache)
-
-
- at unwrap_spec(filename=str)
-def _test_decode_cert(space, filename):
-    cert = libssl_BIO_new(libssl_BIO_s_file())
-    if not cert:
-        raise ssl_error(space, "Can't malloc memory to read file")
-
-    try:
-        if libssl_BIO_read_filename(cert, filename) <= 0:
-            raise ssl_error(space, "Can't open file")
-
-        x = libssl_PEM_read_bio_X509_AUX(cert, None, None, None)
-        if not x:
-            raise ssl_error(space, "Error decoding PEM-encoded file")
-
-        try:
-            return _decode_certificate(space, x)
-        finally:
-            libssl_X509_free(x)
-    finally:
-        libssl_BIO_free(cert)
-
-
-# Data structure for the password callbacks
-class PasswordInfo(object):
-    w_callable = None
-    password = None
-    operationerror = None
-PWINFO_STORAGE = {}
-
-def _password_callback(buf, size, rwflag, userdata):
-    index = rffi.cast(lltype.Signed, userdata)
-    pw_info = PWINFO_STORAGE.get(index, None)
-    if not pw_info:
-        return rffi.cast(rffi.INT, -1)
-    space = pw_info.space
-    password = ""
-    if pw_info.w_callable:
-        try:
-            w_result = space.call_function(pw_info.w_callable)
-            if space.isinstance_w(w_result, space.w_unicode):
-                password = space.str_w(w_result)
-            else:
-                try:
-                    password = pw_info.space.bufferstr_w(w_result)
-                except OperationError as e:
-                    if not e.match(space, space.w_TypeError):
-                        raise
-                    raise oefmt(space.w_TypeError,
-                                "password callback must return a string")
-        except OperationError as e:
-            pw_info.operationerror = e
-            return rffi.cast(rffi.INT, -1)
-    else:
-        password = pw_info.password
-    size = widen(size)
-    if len(password) > size:
-        pw_info.operationerror = oefmt(
-            space.w_ValueError,
-            "password cannot be longer than %d bytes", size)
-        return rffi.cast(rffi.INT, -1)
-    for i, c in enumerate(password):
-        buf[i] = c
-    return rffi.cast(rffi.INT, len(password))
-
-class ServernameCallback(object):
-    w_ctx = None
-    space = None
-SERVERNAME_CALLBACKS = RWeakValueDictionary(int, ServernameCallback)
-
-def _servername_callback(ssl, ad, arg):
-    struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg))
-    w_ctx = struct.w_ctx
-    space = struct.space
-    w_callback = struct.w_set_hostname
-    if not w_ctx.servername_callback:
-        # Possible race condition.
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    # The high-level ssl.SSLSocket object
-    index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl))
-    w_ssl = SOCKET_STORAGE.get(index)
-    assert isinstance(w_ssl, SSLSocket)
-    # The servername callback expects an argument that represents the current
-    # SSL connection and that has a .context attribute that can be changed to
-    # identify the requested hostname. Since the official API is the Python
-    # level API we want to pass the callback a Python level object rather than
-    # a _ssl.SSLSocket instance. If there's an "owner" (typically an
-    # SSLObject) that will be passed. Otherwise if there's a socket then that
-    # will be passed. If both do not exist only then the C-level object is
-    # passed.
-    if w_ssl.w_owner is not None:
-        w_ssl_socket = w_ssl.w_owner()
-    elif w_ssl.w_socket is not None:
-        w_ssl_socket = w_ssl.w_socket()
-    else:
-        w_ssl_socket = w_ssl
-    if space.is_none(w_ssl_socket):
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-    servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name)
-    try:
-        if not servername:
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket, space.w_None, w_ctx)
-
-        else:
-            w_servername = space.newbytes(rffi.charp2str(servername))
-            try:
-                w_servername_idna = space.call_method(
-                    w_servername, 'decode', space.wrap('idna'))
-            except OperationError as e:
-                e.write_unraisable(space, "undecodable server name")
-                ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-                return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-            w_result = space.call_function(w_callback,
-                                           w_ssl_socket,
-                                           w_servername_idna, w_ctx)
-    except OperationError as e:
-        e.write_unraisable(space, "in servername callback")
-        ad[0] = rffi.cast(rffi.INT, SSL_AD_HANDSHAKE_FAILURE)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-    if space.is_none(w_result):
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
-    else:
-        try:
-            ad[0] = rffi.cast(rffi.INT, space.int_w(w_result))
-        except OperationError as e:
-            e.write_unraisable(space, "servername callback result")
-            ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR)
-        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL)
-
-
-class SSLContext(W_Root):
-    ctx = lltype.nullptr(SSL_CTX.TO)
-
-    def __init__(self, space, protocol):
-        if protocol == PY_SSL_VERSION_TLS:
-            method = libssl_TLS_method()
-        elif protocol == PY_SSL_VERSION_TLS1:
-            method = libssl_TLSv1_method()
-        elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
-            method = libssl_SSLv3_method()
-        elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
-            method = libssl_SSLv2_method()
-        elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2:
-            method = libssl_TLSv1_1_method()
-        elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2:
-            method = libssl_TLSv1_2_method()
-        else:
-            raise oefmt(space.w_ValueError, "invalid protocol version")
-        self.ctx = libssl_SSL_CTX_new(method)
-        if not self.ctx:
-            raise ssl_error(space, "failed to allocate SSL context")
-
-        rgc.add_memory_pressure(10 * 1024 * 1024)
-        self.check_hostname = False
-        self.register_finalizer(space)
-
-        # Defaults
-        libssl_SSL_CTX_set_verify(self.ctx, SSL_VERIFY_NONE, None)
-        options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-        if protocol != PY_SSL_VERSION_SSL2:
-            options |= SSL_OP_NO_SSLv2
-        if protocol != PY_SSL_VERSION_SSL3:
-            options |= SSL_OP_NO_SSLv3
-        libssl_SSL_CTX_set_options(self.ctx, options)
-        libssl_SSL_CTX_set_session_id_context(self.ctx, "Python", len("Python"))
-
-        if not OPENSSL_NO_ECDH:
-            # Allow automatic ECDH curve selection (on
-            # OpenSSL 1.0.2+), or use prime256v1 by default.
-            # This is Apache mod_ssl's initialization
-            # policy, so we should be safe.
-            # OpenSSL 1.1 has it enabled by default.
-            if libssl_SSL_CTX_set_ecdh_auto:
-                libssl_SSL_CTX_set_ecdh_auto(self.ctx, 1)
-            else:
-                key = libssl_EC_KEY_new_by_curve_name(NID_X9_62_prime256v1)
-                if not key:
-                    raise _ssl_seterror(space, None, 0)
-                try:
-                    libssl_SSL_CTX_set_tmp_ecdh(self.ctx, key)
-                finally:
-                    libssl_EC_KEY_free(key)
-
-    def _finalize_(self):
-        ctx = self.ctx
-        if ctx:
-            self.ctx = lltype.nullptr(SSL_CTX.TO)
-            libssl_SSL_CTX_free(ctx)
-
-    @staticmethod
-    @unwrap_spec(protocol=int)
-    def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_TLS):
-        self = space.allocate_instance(SSLContext, w_subtype)
-        self.__init__(space, protocol)
-        return space.wrap(self)
-
-    @unwrap_spec(cipherlist=str)
-    def set_ciphers_w(self, space, cipherlist):
-        ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist)
-        if ret == 0:
-            # Clearing the error queue is necessary on some OpenSSL
-            # versions, otherwise the error will be reported again
-            # when another SSL call is done.
-            libssl_ERR_clear_error()
-            raise ssl_error(space, "No cipher can be selected.")
-
-    @unwrap_spec(server_side=int)
-    def wrap_socket_w(self, space, w_sock, server_side,
-                      w_server_hostname=None):
-        assert w_sock is not None
-        # server_hostname is either None (or absent), or to be encoded
-        # using the idna encoding.
-        if space.is_none(w_server_hostname):
-            hostname = None
-        else:
-            hostname = space.bytes_w(
-                space.call_method(w_server_hostname,
-                                  "encode", space.wrap("idna")))
-
-        if hostname and not HAS_SNI:
-            raise oefmt(space.w_ValueError,
-                        "server_hostname is not supported by your OpenSSL "
-                        "library")
-
-        return new_sslobject(space, self, w_sock, server_side, hostname)
-
-    def session_stats_w(self, space):
-        w_stats = space.newdict()
-        for name, ssl_func in SSL_CTX_STATS:
-            w_value = space.wrap(ssl_func(self.ctx))
-            space.setitem_str(w_stats, name, w_value)
-        return w_stats
-
-    def descr_set_default_verify_paths(self, space):
-        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
-            raise ssl_error(space, "")
-
-    def descr_get_options(self, space):
-        return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
-
-    def descr_set_options(self, space, w_new_opts):
-        new_opts = space.int_w(w_new_opts)
-        opts = libssl_SSL_CTX_get_options(self.ctx)
-        clear = opts & ~new_opts
-        set = ~opts & new_opts
-        if clear:
-            if HAVE_SSL_CTX_CLEAR_OPTIONS:
-                libssl_SSL_CTX_clear_options(self.ctx, clear)
-            else:
-                raise oefmt(space.w_ValueError,
-                            "can't clear options before OpenSSL 0.9.8m")
-        if set:
-            libssl_SSL_CTX_set_options(self.ctx, set)
-
-    def descr_get_verify_mode(self, space):
-        mode = libssl_SSL_CTX_get_verify_mode(self.ctx)
-        if mode == SSL_VERIFY_NONE:
-            return space.newlong(PY_SSL_CERT_NONE)
-        elif mode == SSL_VERIFY_PEER:
-            return space.newlong(PY_SSL_CERT_OPTIONAL)
-        elif mode == SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
-            return space.newlong(PY_SSL_CERT_REQUIRED)
-        raise ssl_error(space, "invalid return value from SSL_CTX_get_verify_mode")
-
-    def descr_set_verify_mode(self, space, w_mode):
-        n = space.int_w(w_mode)
-        if n == PY_SSL_CERT_NONE:
-            mode = SSL_VERIFY_NONE
-        elif n == PY_SSL_CERT_OPTIONAL:
-            mode = SSL_VERIFY_PEER
-        elif n == PY_SSL_CERT_REQUIRED:
-            mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT
-        else:
-            raise oefmt(space.w_ValueError,
-                        "invalid value for verify_mode")
-        if mode == SSL_VERIFY_NONE and self.check_hostname:
-            raise oefmt(space.w_ValueError,
-                        "Cannot set verify_mode to CERT_NONE when "
-                        "check_hostname is enabled.")
-        libssl_SSL_CTX_set_verify(self.ctx, mode, None)
-
-    def descr_get_verify_flags(self, space):
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        param = libssl_X509_STORE_get0_param(store)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
-        return space.wrap(flags)
-
-    def descr_set_verify_flags(self, space, w_obj):
-        new_flags = space.int_w(w_obj)
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        param = libssl_X509_STORE_get0_param(store)
-        flags = libssl_X509_VERIFY_PARAM_get_flags(param)
-        flags_clear = flags & ~new_flags
-        flags_set = ~flags & new_flags
-        if flags_clear and not libssl_X509_VERIFY_PARAM_clear_flags(
-                param, flags_clear):
-            raise _ssl_seterror(space, None, 0)
-        if flags_set and not libssl_X509_VERIFY_PARAM_set_flags(
-                param, flags_set):
-            raise _ssl_seterror(space, None, 0)
-
-    def descr_get_check_hostname(self, space):
-        return space.newbool(self.check_hostname)
-
-    def descr_set_check_hostname(self, space, w_obj):
-        check_hostname = space.is_true(w_obj)
-        if check_hostname and libssl_SSL_CTX_get_verify_mode(self.ctx) == SSL_VERIFY_NONE:
-            raise oefmt(space.w_ValueError,
-                        "check_hostname needs a SSL context with either "
-                        "CERT_OPTIONAL or CERT_REQUIRED")
-        self.check_hostname = check_hostname
-
-    def load_cert_chain_w(self, space, w_certfile, w_keyfile=None,
-                          w_password=None):
-        if space.is_none(w_certfile):
-            certfile = None
-        else:
-            certfile = space.str_w(w_certfile)
-        if space.is_none(w_keyfile):
-            keyfile = certfile
-        else:
-            keyfile = space.str_w(w_keyfile)
-        pw_info = PasswordInfo()
-        pw_info.space = space
-        index = -1
-        if not space.is_none(w_password):
-            index = rthread.get_ident()
-            PWINFO_STORAGE[index] = pw_info
-
-            if space.is_true(space.callable(w_password)):
-                pw_info.w_callable = w_password
-            else:
-                if space.isinstance_w(w_password, space.w_unicode):
-                    pw_info.password = space.str_w(w_password)
-                else:
-                    try:
-                        pw_info.password = space.bufferstr_w(w_password)
-                    except OperationError as e:
-                        if not e.match(space, space.w_TypeError):
-                            raise
-                        raise oefmt(space.w_TypeError,
-                                    "password should be a string or callable")
-
-            libssl_SSL_CTX_set_default_passwd_cb(
-                self.ctx, _password_callback)
-            libssl_SSL_CTX_set_default_passwd_cb_userdata(
-                self.ctx, rffi.cast(rffi.VOIDP, index))
-
-        try:
-            ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile)
-            if ret != 1:
-                if pw_info.operationerror:
-                    libssl_ERR_clear_error()
-                    raise pw_info.operationerror
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-            ret = libssl_SSL_CTX_use_PrivateKey_file(self.ctx, keyfile,
-                                                     SSL_FILETYPE_PEM)
-            if ret != 1:
-                if pw_info.operationerror:
-                    libssl_ERR_clear_error()
-                    raise pw_info.operationerror
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-            ret = libssl_SSL_CTX_check_private_key(self.ctx)
-            if ret != 1:
-                raise _ssl_seterror(space, None, -1)
-        finally:
-            if index >= 0:
-                del PWINFO_STORAGE[index]
-            libssl_SSL_CTX_set_default_passwd_cb(
-                self.ctx, lltype.nullptr(pem_password_cb.TO))
-            libssl_SSL_CTX_set_default_passwd_cb_userdata(
-                self.ctx, None)
-
-    @unwrap_spec(filepath='fsencode')
-    def load_dh_params_w(self, space, filepath):
-        bio = libssl_BIO_new_file(filepath, "r")
-        if not bio:
-            errno = get_saved_errno()
-            libssl_ERR_clear_error()
-            raise wrap_oserror(space, OSError(errno, ''),
-                               exception_name = 'w_IOError')
-        try:
-            dh = libssl_PEM_read_bio_DHparams(bio, None, None, None)
-        finally:
-            libssl_BIO_free(bio)
-        if not dh:
-            errno = get_saved_errno()
-            if errno != 0:
-                libssl_ERR_clear_error()
-                raise wrap_oserror(space, OSError(errno, ''))
-            else:
-                raise _ssl_seterror(space, None, 0)
-        try:
-            if libssl_SSL_CTX_set_tmp_dh(self.ctx, dh) == 0:
-                raise _ssl_seterror(space, None, 0)
-        finally:
-            libssl_DH_free(dh)
-
-    def load_verify_locations_w(self, space, w_cafile=None, w_capath=None,
-                                w_cadata=None):
-        if space.is_none(w_cafile):
-            cafile = None
-        else:
-            cafile = space.str_w(w_cafile)
-        if space.is_none(w_capath):
-            capath = None
-        else:
-            capath = space.str_w(w_capath)
-        if space.is_none(w_cadata):
-            cadata = None
-            ca_file_type = -1
-        else:
-            if not space.isinstance_w(w_cadata, space.w_unicode):
-                ca_file_type = SSL_FILETYPE_ASN1
-                cadata = space.bufferstr_w(w_cadata)
-            else:
-                ca_file_type = SSL_FILETYPE_PEM
-                try:
-                    cadata = space.unicode_w(w_cadata).encode('ascii')
-                except UnicodeEncodeError:
-                    raise oefmt(space.w_TypeError,
-                                "cadata should be a ASCII string or a "
-                                "bytes-like object")
-        if cafile is None and capath is None and cadata is None:
-            raise oefmt(space.w_TypeError,
-                        "cafile and capath cannot be both omitted")
-        # load from cadata
-        if cadata is not None:
-            with rffi.scoped_nonmovingbuffer(cadata) as buf:
-                self._add_ca_certs(space, buf, len(cadata), ca_file_type)
-
-        # load cafile or capath
-        if cafile is not None or capath is not None:
-            ret = libssl_SSL_CTX_load_verify_locations(
-                self.ctx, cafile, capath)
-            if ret != 1:
-                errno = get_saved_errno()
-                if errno:
-                    libssl_ERR_clear_error()
-                    raise wrap_oserror(space, OSError(errno, ''),
-                                       exception_name = 'w_IOError')
-                else:
-                    raise _ssl_seterror(space, None, -1)
-
-    def _add_ca_certs(self, space, data, size, ca_file_type):
-        biobuf = libssl_BIO_new_mem_buf(data, size)
-        if not biobuf:
-            raise ssl_error(space, "Can't allocate buffer")
-        try:
-            store = libssl_SSL_CTX_get_cert_store(self.ctx)
-            loaded = 0
-            while True:
-                if ca_file_type == SSL_FILETYPE_ASN1:
-                    cert = libssl_d2i_X509_bio(
-                        biobuf, None)
-                else:
-                    cert = libssl_PEM_read_bio_X509(
-                        biobuf, None, None, None)
-                if not cert:
-                    break
-                try:
-                    r = libssl_X509_STORE_add_cert(store, cert)
-                finally:
-                    libssl_X509_free(cert)
-                if not r:
-                    err = libssl_ERR_peek_last_error()
-                    if (libssl_ERR_GET_LIB(err) == ERR_LIB_X509 and
-                        libssl_ERR_GET_REASON(err) ==
-                        X509_R_CERT_ALREADY_IN_HASH_TABLE):
-                        # cert already in hash table, not an error
-                        libssl_ERR_clear_error()
-                    else:
-                        break
-                loaded += 1
-
-            err = libssl_ERR_peek_last_error()
-            if (ca_file_type == SSL_FILETYPE_ASN1 and
-                loaded > 0 and
-                libssl_ERR_GET_LIB(err) == ERR_LIB_ASN1 and
-                libssl_ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG):
-                # EOF ASN1 file, not an error
-                libssl_ERR_clear_error()
-            elif (ca_file_type == SSL_FILETYPE_PEM and
-                  loaded > 0 and
-                  libssl_ERR_GET_LIB(err) == ERR_LIB_PEM and
-                  libssl_ERR_GET_REASON(err) == PEM_R_NO_START_LINE):
-                # EOF PEM file, not an error
-                libssl_ERR_clear_error()
-            else:
-                raise _ssl_seterror(space, None, 0)
-        finally:
-            libssl_BIO_free(biobuf)
-
-    def cert_store_stats_w(self, space):
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        x509 = 0
-        x509_ca = 0
-        crl = 0
-        objs = libssl_X509_STORE_get0_objects(store)
-        for i in range(libssl_sk_X509_OBJECT_num(objs)):
-            obj = libssl_sk_X509_OBJECT_value(objs, i)
-            obj_type = intmask(libssl_X509_OBJECT_get_type(obj))
-            if obj_type == X509_LU_X509:
-                x509 += 1
-                if libssl_X509_check_ca(
-                        libssl_X509_OBJECT_get0_X509(obj)):
-                    x509_ca += 1
-            elif obj_type == X509_LU_CRL:
-                crl += 1
-            else:
-                # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
-                # As far as I can tell they are internal states and never
-                # stored in a cert store
-                pass
-        w_result = space.newdict()
-        space.setitem(w_result,
-                      space.wrap('x509'), space.wrap(x509))
-        space.setitem(w_result,
-                      space.wrap('x509_ca'), space.wrap(x509_ca))
-        space.setitem(w_result,
-                      space.wrap('crl'), space.wrap(crl))
-        return w_result
-
-    @unwrap_spec(protos='bufferstr')
-    def set_npn_protocols_w(self, space, protos):
-        if not HAS_NPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The NPN extension requires OpenSSL 1.0.1 or later.")
-
-        self.npn_protocols = SSLNpnProtocols(self.ctx, protos)
-
-    @unwrap_spec(protos='bufferstr')
-    def set_alpn_protocols_w(self, space, protos):
-        if not HAS_ALPN:
-            raise oefmt(space.w_NotImplementedError,
-                        "The ALPN extension requires OpenSSL 1.0.2 or later.")
-
-        self.alpn_protocols = SSLAlpnProtocols(self.ctx, protos)
-
-    def get_ca_certs_w(self, space, w_binary_form=None):
-        if w_binary_form and space.is_true(w_binary_form):
-            binary_mode = True
-        else:
-            binary_mode = False
-        rlist = []
-        store = libssl_SSL_CTX_get_cert_store(self.ctx)
-        objs = libssl_X509_STORE_get0_objects(store)
-        for i in range(libssl_sk_X509_OBJECT_num(objs)):
-            obj = libssl_sk_X509_OBJECT_value(objs, i)
-            if intmask(libssl_X509_OBJECT_get_type(obj)) != X509_LU_X509:
-                # not a x509 cert
-                continue
-            # CA for any purpose
-            cert = libssl_X509_OBJECT_get0_X509(obj)
-            if not libssl_X509_check_ca(cert):
-                continue
-            if binary_mode:
-                rlist.append(_certificate_to_der(space, cert))
-            else:
-                rlist.append(_decode_certificate(space, cert))
-        return space.newlist(rlist)
-
-    @unwrap_spec(name=str)
-    def set_ecdh_curve_w(self, space, name):
-        nid = libssl_OBJ_sn2nid(name)
-        if nid == 0:
-            raise oefmt(space.w_ValueError,
-                        "unknown elliptic curve name '%s'", name)
-        key = libssl_EC_KEY_new_by_curve_name(nid)
-        if not key:
-            raise _ssl_seterror(space, None, 0)
-        try:
-            libssl_SSL_CTX_set_tmp_ecdh(self.ctx, key)
-        finally:
-            libssl_EC_KEY_free(key)
-
-    def set_servername_callback_w(self, space, w_callback):
-        if space.is_none(w_callback):
-            libssl_SSL_CTX_set_tlsext_servername_callback(
-                self.ctx, lltype.nullptr(servername_cb.TO))
-            self.servername_callback = None
-            return
-        if not space.is_true(space.callable(w_callback)):
-            raise oefmt(space.w_TypeError, "not a callable object")
-        callback_struct = ServernameCallback()
-        callback_struct.space = space
-        callback_struct.w_ctx = self
-        callback_struct.w_set_hostname = w_callback
-        self.servername_callback = callback_struct
-        index = compute_unique_id(self)
-        SERVERNAME_CALLBACKS.set(index, callback_struct)
-        libssl_SSL_CTX_set_tlsext_servername_callback(
-            self.ctx, _servername_callback)
-        libssl_SSL_CTX_set_tlsext_servername_arg(self.ctx,
-                                                 rffi.cast(rffi.VOIDP, index))
-
-SSLContext.typedef = TypeDef(
-    "_ssl._SSLContext",
-    __new__ = interp2app(SSLContext.descr_new),
-    _wrap_socket = interp2app(SSLContext.wrap_socket_w),
-    set_ciphers = interp2app(SSLContext.set_ciphers_w),
-    load_cert_chain = interp2app(SSLContext.load_cert_chain_w),
-    load_verify_locations = interp2app(SSLContext.load_verify_locations_w),
-    session_stats = interp2app(SSLContext.session_stats_w),
-    cert_store_stats=interp2app(SSLContext.cert_store_stats_w),
-    load_dh_params=interp2app(SSLContext.load_dh_params_w),
-    set_default_verify_paths=interp2app(SSLContext.descr_set_default_verify_paths),
-    _set_npn_protocols=interp2app(SSLContext.set_npn_protocols_w),
-    _set_alpn_protocols=interp2app(SSLContext.set_alpn_protocols_w),
-    get_ca_certs=interp2app(SSLContext.get_ca_certs_w),
-    set_ecdh_curve=interp2app(SSLContext.set_ecdh_curve_w),
-    set_servername_callback=interp2app(SSLContext.set_servername_callback_w),
-
-    options=GetSetProperty(SSLContext.descr_get_options,
-                           SSLContext.descr_set_options),
-    verify_mode=GetSetProperty(SSLContext.descr_get_verify_mode,
-                               SSLContext.descr_set_verify_mode),
-    verify_flags=GetSetProperty(SSLContext.descr_get_verify_flags,
-                                SSLContext.descr_set_verify_flags),
-    check_hostname=GetSetProperty(SSLContext.descr_get_check_hostname,
-                                  SSLContext.descr_set_check_hostname),
-)
-
-
-def _asn1obj2py(space, obj):
-    nid = libssl_OBJ_obj2nid(obj)
-    if nid == NID_undef:
-        raise oefmt(space.w_ValueError, "Unknown object")
-    with rffi.scoped_alloc_buffer(100) as buf:
-        buflen = libssl_OBJ_obj2txt(buf.raw, 100, obj, 1)
-        if buflen < 0:
-            raise _ssl_seterror(space, None, 0)
-        if buflen:
-            w_buf = space.wrap(buf.str(buflen))
-        else:
-            w_buf = space.w_None
-    w_sn = space.wrap(rffi.charp2str(libssl_OBJ_nid2sn(nid)))
-    w_ln = space.wrap(rffi.charp2str(libssl_OBJ_nid2ln(nid)))
-    return space.newtuple([space.wrap(nid), w_sn, w_ln, w_buf])
-
-
- at unwrap_spec(txt=str, name=bool)
-def txt2obj(space, txt, name=False):
-    obj = libssl_OBJ_txt2obj(txt, not name)
-    if not obj:
-        raise oefmt(space.w_ValueError, "unknown object '%s'", txt)
-    try:
-        w_result = _asn1obj2py(space, obj)
-    finally:
-        libssl_ASN1_OBJECT_free(obj)
-    return w_result
-
-
- at unwrap_spec(nid=int)
-def nid2obj(space, nid):
-    if nid < NID_undef:
-        raise oefmt(space.w_ValueError, "NID must be positive")
-    obj = libssl_OBJ_nid2obj(nid)
-    if not obj:
-        raise oefmt(space.w_ValueError, "unknown NID %d", nid)
-    try:
-        w_result = _asn1obj2py(space, obj)
-    finally:
-        libssl_ASN1_OBJECT_free(obj)
-    return w_result
-
-
-def w_convert_path(space, path):
-    if not path:
-        return space.w_None
-    else:
-        return fsdecode(space, space.newbytes(rffi.charp2str(path)))
-
-def get_default_verify_paths(space):
-    return space.newtuple([
-        w_convert_path(space, libssl_X509_get_default_cert_file_env()),
-        w_convert_path(space, libssl_X509_get_default_cert_file()),
-        w_convert_path(space, libssl_X509_get_default_cert_dir_env()),
-        w_convert_path(space, libssl_X509_get_default_cert_dir()),
-        ])
-
-
-class MemoryBIO(W_Root):
-    pass
-MemoryBIO.typedef = TypeDef(
-    "_ssl.MemoryBIO",
-)
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
deleted file mode 100644
--- a/pypy/module/_ssl/interp_win32.py
+++ /dev/null
@@ -1,180 +0,0 @@
-from rpython.rlib import rwin32
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rtyper.tool import rffi_platform
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.interpreter.gateway import unwrap_spec

From pypy.commits at gmail.com  Thu Nov 17 08:29:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 05:29:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: rename openssl to _cffi_ssl
Message-ID: <582db0cf.0c20190a.3ea2e.2ff1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88437:032895d1e16c
Date: 2016-11-17 14:25 +0100
http://bitbucket.org/pypy/pypy/changeset/032895d1e16c/

Log:	rename openssl to _cffi_ssl

diff too long, truncating to 2000 out of 18999 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -78,9 +78,3 @@
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
-^lib_pypy/_.*cffi.*.c$
-^lib_pypy/_.*cffi.*.o$
-^lib_pypy/_.*cffi.*.so$
-^lib_pypy/_openssl.c$
-^lib_pypy/_openssl.o$
-^lib_pypy/_openssl.*.so$
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Vim
+
+*.swp
+*.swo
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,8 @@
+# PyPy's SSL module
+
+Most of the CFFI code is copied from cryptography
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..180c02ff82d3363f34a334aae22c9876d4c96481
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "tls1",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;
+
+typedef enum {
+    kSSLSessionOptionBreakOnServerAuth,
+    kSSLSessionOptionBreakOnCertRequested,
+} SSLSessionOption;
+
+typedef enum {
+    kNeverAuthenticate,
+    kAlwaysAuthenticate,
+    kTryAuthenticate
+} SSLAuthenticate;
+
+typedef enum {
+    kSSLIdle,
+    kSSLHandshake,
+    kSSLConnected,
+    kSSLClosed,
+    kSSLAborted
+} SSLSessionState;
+
+typedef enum {
+    kSSLProtocolUnknown = 0,
+    kSSLProtocol3       = 2,
+    kTLSProtocol1       = 4,
+    /* DEPRECATED on iOS */
+    kSSLProtocol2       = 1,
+    kSSLProtocol3Only   = 3,
+    kTLSProtocol1Only   = 5,
+    kSSLProtocolAll     = 6,
+} SSLProtocol;
+
+typedef UInt32 SSLCipherSuite;
+enum {
+   SSL_NULL_WITH_NULL_NULL =               0x0000,
+   SSL_RSA_WITH_NULL_MD5 =                 0x0001,
+   SSL_RSA_WITH_NULL_SHA =                 0x0002,
+   SSL_RSA_EXPORT_WITH_RC4_40_MD5 =        0x0003,
+   SSL_RSA_WITH_RC4_128_MD5 =              0x0004,
+   SSL_RSA_WITH_RC4_128_SHA =              0x0005,
+   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 =    0x0006,
+   SSL_RSA_WITH_IDEA_CBC_SHA =             0x0007,
+   SSL_RSA_EXPORT_WITH_DES40_CBC_SHA =     0x0008,
+   SSL_RSA_WITH_DES_CBC_SHA =              0x0009,
+   SSL_RSA_WITH_3DES_EDE_CBC_SHA =         0x000A,
+   SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA =  0x000B,
+   SSL_DH_DSS_WITH_DES_CBC_SHA =           0x000C,
+   SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA =      0x000D,
+   SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA =  0x000E,
+   SSL_DH_RSA_WITH_DES_CBC_SHA =           0x000F,
+   SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA =      0x0010,
+   SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011,
+   SSL_DHE_DSS_WITH_DES_CBC_SHA =          0x0012,
+   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA =     0x0013,
+   SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014,
+   SSL_DHE_RSA_WITH_DES_CBC_SHA =          0x0015,
+   SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA =     0x0016,
+   SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 =    0x0017,
+   SSL_DH_anon_WITH_RC4_128_MD5 =          0x0018,
+   SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019,
+   SSL_DH_anon_WITH_DES_CBC_SHA =          0x001A,
+   SSL_DH_anon_WITH_3DES_EDE_CBC_SHA =     0x001B,
+   SSL_FORTEZZA_DMS_WITH_NULL_SHA =        0x001C,
+   SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D,
+
+   /* TLS addenda using AES, per RFC 3268 */
+   TLS_RSA_WITH_AES_128_CBC_SHA      =     0x002F,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA   =     0x0030,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA   =     0x0031,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA  =     0x0032,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA  =     0x0033,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA  =     0x0034,
+   TLS_RSA_WITH_AES_256_CBC_SHA      =     0x0035,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA   =     0x0036,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA   =     0x0037,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA  =     0x0038,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA  =     0x0039,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA  =     0x003A,
+
+   /* ECDSA addenda, RFC 4492 */
+   TLS_ECDH_ECDSA_WITH_NULL_SHA           =    0xC001,
+   TLS_ECDH_ECDSA_WITH_RC4_128_SHA        =    0xC002,
+   TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA   =    0xC003,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA    =    0xC004,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA    =    0xC005,
+   TLS_ECDHE_ECDSA_WITH_NULL_SHA          =    0xC006,
+   TLS_ECDHE_ECDSA_WITH_RC4_128_SHA       =    0xC007,
+   TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA  =    0xC008,
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA   =    0xC009,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA   =    0xC00A,
+   TLS_ECDH_RSA_WITH_NULL_SHA             =    0xC00B,
+   TLS_ECDH_RSA_WITH_RC4_128_SHA          =    0xC00C,
+   TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA     =    0xC00D,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA      =    0xC00E,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA      =    0xC00F,
+   TLS_ECDHE_RSA_WITH_NULL_SHA            =    0xC010,
+   TLS_ECDHE_RSA_WITH_RC4_128_SHA         =    0xC011,
+   TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA    =    0xC012,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA     =    0xC013,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA     =    0xC014,
+   TLS_ECDH_anon_WITH_NULL_SHA            =    0xC015,
+   TLS_ECDH_anon_WITH_RC4_128_SHA         =    0xC016,
+   TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA    =    0xC017,
+   TLS_ECDH_anon_WITH_AES_128_CBC_SHA     =    0xC018,
+   TLS_ECDH_anon_WITH_AES_256_CBC_SHA     =    0xC019,
+
+   /* TLS 1.2 addenda, RFC 5246 */
+   /* Initial state. */
+   TLS_NULL_WITH_NULL_NULL                   = 0x0000,
+
+   /* Server provided RSA certificate for key exchange. */
+   TLS_RSA_WITH_NULL_MD5                     = 0x0001,
+   TLS_RSA_WITH_NULL_SHA                     = 0x0002,
+   TLS_RSA_WITH_RC4_128_MD5                  = 0x0004,
+   TLS_RSA_WITH_RC4_128_SHA                  = 0x0005,
+   TLS_RSA_WITH_3DES_EDE_CBC_SHA             = 0x000A,
+   TLS_RSA_WITH_NULL_SHA256                  = 0x003B,
+   TLS_RSA_WITH_AES_128_CBC_SHA256           = 0x003C,
+   TLS_RSA_WITH_AES_256_CBC_SHA256           = 0x003D,
+
+   /* Server-authenticated (and optionally client-authenticated)
+      Diffie-Hellman. */
+   TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA          = 0x000D,
+   TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA          = 0x0010,
+   TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA         = 0x0013,
+   TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA         = 0x0016,
+   TLS_DH_DSS_WITH_AES_128_CBC_SHA256        = 0x003E,
+   TLS_DH_RSA_WITH_AES_128_CBC_SHA256        = 0x003F,
+   TLS_DHE_DSS_WITH_AES_128_CBC_SHA256       = 0x0040,
+   TLS_DHE_RSA_WITH_AES_128_CBC_SHA256       = 0x0067,
+   TLS_DH_DSS_WITH_AES_256_CBC_SHA256        = 0x0068,
+   TLS_DH_RSA_WITH_AES_256_CBC_SHA256        = 0x0069,
+   TLS_DHE_DSS_WITH_AES_256_CBC_SHA256       = 0x006A,
+   TLS_DHE_RSA_WITH_AES_256_CBC_SHA256       = 0x006B,
+
+   /* Completely anonymous Diffie-Hellman */
+   TLS_DH_anon_WITH_RC4_128_MD5              = 0x0018,
+   TLS_DH_anon_WITH_3DES_EDE_CBC_SHA         = 0x001B,
+   TLS_DH_anon_WITH_AES_128_CBC_SHA256       = 0x006C,
+   TLS_DH_anon_WITH_AES_256_CBC_SHA256       = 0x006D,
+
+   /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites
+      for TLS. */
+   TLS_RSA_WITH_AES_128_GCM_SHA256           = 0x009C,
+   TLS_RSA_WITH_AES_256_GCM_SHA384           = 0x009D,
+   TLS_DHE_RSA_WITH_AES_128_GCM_SHA256       = 0x009E,
+   TLS_DHE_RSA_WITH_AES_256_GCM_SHA384       = 0x009F,
+   TLS_DH_RSA_WITH_AES_128_GCM_SHA256        = 0x00A0,
+   TLS_DH_RSA_WITH_AES_256_GCM_SHA384        = 0x00A1,
+   TLS_DHE_DSS_WITH_AES_128_GCM_SHA256       = 0x00A2,
+   TLS_DHE_DSS_WITH_AES_256_GCM_SHA384       = 0x00A3,
+   TLS_DH_DSS_WITH_AES_128_GCM_SHA256        = 0x00A4,
+   TLS_DH_DSS_WITH_AES_256_GCM_SHA384        = 0x00A5,
+   TLS_DH_anon_WITH_AES_128_GCM_SHA256       = 0x00A6,
+   TLS_DH_anon_WITH_AES_256_GCM_SHA384       = 0x00A7,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      HMAC SHA-256/384. */
+   TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256   = 0xC023,
+   TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384   = 0xC024,
+   TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256    = 0xC025,
+   TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384    = 0xC026,
+   TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256     = 0xC027,
+   TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384     = 0xC028,
+   TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256      = 0xC029,
+   TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384      = 0xC02A,
+
+   /* Addenda from rfc 5289  Elliptic Curve Cipher Suites with
+      SHA-256/384 and AES Galois Counter Mode (GCM) */
+   TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256   = 0xC02B,
+   TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384   = 0xC02C,
+   TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256    = 0xC02D,
+   TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384    = 0xC02E,
+   TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256     = 0xC02F,
+   TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384     = 0xC030,
+   TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256      = 0xC031,
+   TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384      = 0xC032,
+
+   /* RFC 5746 - Secure Renegotiation */
+   TLS_EMPTY_RENEGOTIATION_INFO_SCSV         = 0x00FF,
+
+   /*
+   * Tags for SSL 2 cipher kinds that are not specified
+   * for SSL 3.
+   */
+   SSL_RSA_WITH_RC2_CBC_MD5 =              0xFF80,
+   SSL_RSA_WITH_IDEA_CBC_MD5 =             0xFF81,
+   SSL_RSA_WITH_DES_CBC_MD5 =              0xFF82,
+   SSL_RSA_WITH_3DES_EDE_CBC_MD5 =         0xFF83,
+   SSL_NO_SUCH_CIPHERSUITE =               0xFFFF
+};
+
+typedef enum {
+    kSSLClientCertNone,
+    kSSLClientCertRequested,
+    kSSLClientCertSent,
+    kSSLClientCertRejected
+} SSLClientCertificateState;
+
+enum {
+    errSSLProtocol              = -9800,
+    errSSLNegotiation           = -9801,
+    errSSLFatalAlert            = -9802,
+    errSSLWouldBlock            = -9803,
+    errSSLSessionNotFound       = -9804,
+    errSSLClosedGraceful        = -9805,
+    errSSLClosedAbort           = -9806,
+    errSSLXCertChainInvalid     = -9807,
+    errSSLBadCert               = -9808,
+    errSSLCrypto                = -9809,
+    errSSLInternal              = -9810,
+    errSSLModuleAttach          = -9811,
+    errSSLUnknownRootCert       = -9812,
+    errSSLNoRootCert            = -9813,
+    errSSLCertExpired           = -9814,
+    errSSLCertNotYetValid       = -9815,
+    errSSLClosedNoNotify        = -9816,
+    errSSLBufferOverflow        = -9817,
+    errSSLBadCipherSuite        = -9818,
+    errSSLPeerUnexpectedMsg     = -9819,
+    errSSLPeerBadRecordMac      = -9820,
+    errSSLPeerDecryptionFail    = -9821,
+    errSSLPeerRecordOverflow    = -9822,
+    errSSLPeerDecompressFail    = -9823,
+    errSSLPeerHandshakeFail     = -9824,
+    errSSLPeerBadCert           = -9825,
+    errSSLPeerUnsupportedCert   = -9826,
+    errSSLPeerCertRevoked       = -9827,
+    errSSLPeerCertExpired       = -9828,
+    errSSLPeerCertUnknown       = -9829,
+    errSSLIllegalParam          = -9830,
+    errSSLPeerUnknownCA         = -9831,
+    errSSLPeerAccessDenied      = -9832,
+    errSSLPeerDecodeError       = -9833,
+    errSSLPeerDecryptError      = -9834,
+    errSSLPeerExportRestriction = -9835,
+    errSSLPeerProtocolVersion   = -9836,
+    errSSLPeerInsufficientSecurity = -9837,
+    errSSLPeerInternalError     = -9838,
+    errSSLPeerUserCancelled     = -9839,
+    errSSLPeerNoRenegotiation   = -9840,
+    errSSLServerAuthCompleted   = -9841,
+    errSSLClientCertRequested   = -9842,
+    errSSLHostNameMismatch      = -9843,
+    errSSLConnectionRefused     = -9844,
+    errSSLDecryptionFail        = -9845,
+    errSSLBadRecordMac          = -9846,
+    errSSLRecordOverflow        = -9847,
+    errSSLBadConfiguration      = -9848,
+    errSSLLast                  = -9849     /* end of range, to be deleted */
+};
+"""
+
+FUNCTIONS = """
+OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef);
+OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *);
+OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean);
+OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate);
+
+OSStatus SSLHandshake(SSLContextRef);
+OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *);
+OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *);
+OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t);
+OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *);
+OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *);
+OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *);
+OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *);
+OSStatus SSLClose(SSLContextRef);
+
+OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t);
+OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *);
+OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *);
+OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *);
+OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t);
+OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *);
+
+OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean);
+OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *);
+OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *);
+OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef);
+OSStatus SSLGetClientCertificateState(SSLContextRef,
+                                      SSLClientCertificateState *);
+OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust);
+
+OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t);
+OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *);
+OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c
@@ -0,0 +1,22 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a,
+                                            uint8_t *b, size_t len_b) {
+    size_t i = 0;
+    uint8_t mismatch = 0;
+    if (len_a != len_b) {
+        return 0;
+    }
+    for (i = 0; i < len_a; i++) {
+        mismatch |= a[i] ^ b[i];
+    }
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *,
+                                            size_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c
@@ -0,0 +1,65 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+/* Returns the value of the input with the most-significant-bit copied to all
+   of the bits. */
+static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) {
+    return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1;
+}
+
+/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time
+   fashion */
+static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) {
+    a -= b;
+    return Cryptography_DUPLICATE_MSB_TO_ALL(a);
+}
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data,
+                                         uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    for (i = 0; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & (pad_size ^ b));
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
+
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *data,
+                                            uint16_t block_len) {
+    uint16_t i;
+    uint16_t pad_size = data[block_len - 1];
+    uint16_t mismatch = 0;
+    /* Skip the first one with the pad size */
+    for (i = 1; i < block_len; i++) {
+        unsigned int mask = Cryptography_constant_time_lt(i, pad_size);
+        uint16_t b = data[block_len - 1 - i];
+        mismatch |= (mask & b);
+    }
+
+    /* Check to make sure the pad_size was within the valid range. */
+    mismatch |= ~Cryptography_constant_time_lt(0, pad_size);
+    mismatch |= Cryptography_constant_time_lt(block_len, pad_size);
+
+    /* Make sure any bits set are copied to the lowest bit */
+    mismatch |= mismatch >> 8;
+    mismatch |= mismatch >> 4;
+    mismatch |= mismatch >> 2;
+    mismatch |= mismatch >> 1;
+    /* Now check the low bit to see if it's set */
+    return (mismatch & 1) == 0;
+}
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h
@@ -0,0 +1,6 @@
+// This file is dual licensed under the terms of the Apache License, Version
+// 2.0, and the BSD License. See the LICENSE file in the root of this
+// repository for complete details.
+
+uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t);
+uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t);
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py
@@ -0,0 +1,50 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/aes.h>
+"""
+
+TYPES = """
+static const int Cryptography_HAS_AES_WRAP;
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT;
+
+struct aes_key_st {
+    ...;
+};
+typedef struct aes_key_st AES_KEY;
+"""
+
+FUNCTIONS = """
+int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *);
+int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *);
+
+int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                 const unsigned char *, unsigned int);
+int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *,
+                   const unsigned char *, unsigned int);
+"""
+
+MACROS = """
+/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for
+   this in 1.0.1+. */
+void AES_ctr128_encrypt(const unsigned char *, unsigned char *,
+                        size_t, const AES_KEY *, unsigned char[],
+                        unsigned char[], unsigned int *);
+"""
+
+CUSTOMIZATIONS = """
+static const long Cryptography_HAS_AES_WRAP = 1;
+#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER)
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0;
+void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *,
+                           size_t, const AES_KEY *,
+                           unsigned char[], unsigned char[],
+                           unsigned int *) = NULL;
+#else
+static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1;
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py
@@ -0,0 +1,164 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/asn1.h>
+"""
+
+TYPES = """
+typedef int... time_t;
+
+typedef int ASN1_BOOLEAN;
+typedef ... ASN1_INTEGER;
+
+struct asn1_string_st {
+    int length;
+    int type;
+    unsigned char *data;
+    long flags;
+};
+
+typedef struct asn1_string_st ASN1_OCTET_STRING;
+typedef struct asn1_string_st ASN1_IA5STRING;
+typedef struct asn1_string_st ASN1_BIT_STRING;
+typedef struct asn1_string_st ASN1_TIME;
+typedef ... ASN1_OBJECT;
+typedef struct asn1_string_st ASN1_STRING;
+typedef struct asn1_string_st ASN1_UTF8STRING;
+typedef ... ASN1_TYPE;
+typedef ... ASN1_GENERALIZEDTIME;
+typedef ... ASN1_ENUMERATED;
+typedef ... ASN1_ITEM;
+typedef ... ASN1_VALUE;
+
+typedef ... ASN1_ITEM_EXP;
+
+typedef ... ASN1_UTCTIME;
+
+static const int V_ASN1_GENERALIZEDTIME;
+
+static const int MBSTRING_FLAG;
+static const int MBSTRING_ASC;
+static const int MBSTRING_BMP;
+static const int MBSTRING_UTF8;
+static const int MBSTRING_UNIV;
+"""
+
+FUNCTIONS = """
+ASN1_OBJECT *ASN1_OBJECT_new(void);
+void ASN1_OBJECT_free(ASN1_OBJECT *);
+
+/*  ASN1 OBJECT IDENTIFIER */
+ASN1_OBJECT *d2i_ASN1_OBJECT(ASN1_OBJECT **, const unsigned char **, long);
+
+/*  ASN1 STRING */
+ASN1_STRING *ASN1_STRING_new(void);
+ASN1_STRING *ASN1_STRING_type_new(int);
+void ASN1_STRING_free(ASN1_STRING *);
+unsigned char *ASN1_STRING_data(ASN1_STRING *);
+int ASN1_STRING_set(ASN1_STRING *, const void *, int);
+
+/*  ASN1 OCTET STRING */
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_new(void);
+void ASN1_OCTET_STRING_free(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_set(ASN1_OCTET_STRING *, const unsigned char *, int);
+
+/* ASN1 IA5STRING */
+ASN1_IA5STRING *ASN1_IA5STRING_new(void);
+
+/*  ASN1 INTEGER */
+ASN1_INTEGER *ASN1_INTEGER_new(void);
+void ASN1_INTEGER_free(ASN1_INTEGER *);
+int ASN1_INTEGER_set(ASN1_INTEGER *, long);
+
+/*  ASN1 TIME */
+ASN1_TIME *ASN1_TIME_new(void);
+void ASN1_TIME_free(ASN1_TIME *);
+ASN1_GENERALIZEDTIME *ASN1_TIME_to_generalizedtime(ASN1_TIME *,
+                                                   ASN1_GENERALIZEDTIME **);
+ASN1_TIME *ASN1_TIME_set(ASN1_TIME *, time_t);
+
+/*  ASN1 UTCTIME */
+ASN1_UTCTIME *ASN1_UTCTIME_new(void);
+void ASN1_UTCTIME_free(ASN1_UTCTIME *);
+int ASN1_UTCTIME_cmp_time_t(const ASN1_UTCTIME *, time_t);
+ASN1_UTCTIME *ASN1_UTCTIME_set(ASN1_UTCTIME *, time_t);
+
+/*  ASN1 GENERALIZEDTIME */
+int ASN1_GENERALIZEDTIME_set_string(ASN1_GENERALIZEDTIME *, const char *);
+ASN1_GENERALIZEDTIME *ASN1_GENERALIZEDTIME_set(ASN1_GENERALIZEDTIME *, time_t);
+void ASN1_GENERALIZEDTIME_free(ASN1_GENERALIZEDTIME *);
+int i2d_ASN1_GENERALIZEDTIME(ASN1_GENERALIZEDTIME *, unsigned char **);
+
+/*  ASN1 ENUMERATED */
+ASN1_ENUMERATED *ASN1_ENUMERATED_new(void);
+void ASN1_ENUMERATED_free(ASN1_ENUMERATED *);
+int ASN1_ENUMERATED_set(ASN1_ENUMERATED *, long);
+int i2d_ASN1_ENUMERATED(ASN1_ENUMERATED *, unsigned char **);
+
+ASN1_VALUE *ASN1_item_d2i(ASN1_VALUE **, const unsigned char **, long,
+                          const ASN1_ITEM *);
+int ASN1_BIT_STRING_set_bit(ASN1_BIT_STRING *, int, int);
+"""
+
+MACROS = """
+/* These became const ASN1_* in 1.1.0 */
+int i2d_ASN1_OBJECT(ASN1_OBJECT *, unsigned char **);
+int ASN1_STRING_type(ASN1_STRING *);
+int ASN1_STRING_to_UTF8(unsigned char **, ASN1_STRING *);
+long ASN1_ENUMERATED_get(ASN1_ENUMERATED *);
+int i2a_ASN1_INTEGER(BIO *, ASN1_INTEGER *);
+
+ASN1_UTF8STRING *ASN1_UTF8STRING_new(void);
+void ASN1_UTF8STRING_free(ASN1_UTF8STRING *);
+
+ASN1_BIT_STRING *ASN1_BIT_STRING_new(void);
+void ASN1_BIT_STRING_free(ASN1_BIT_STRING *);
+int i2d_ASN1_BIT_STRING(ASN1_BIT_STRING *, unsigned char **);
+int i2d_ASN1_OCTET_STRING(ASN1_OCTET_STRING *, unsigned char **);
+int i2d_ASN1_INTEGER(ASN1_INTEGER *, unsigned char **);
+/* This is not a macro, but is const on some versions of OpenSSL */
+int ASN1_BIT_STRING_get_bit(ASN1_BIT_STRING *, int);
+ASN1_TIME *M_ASN1_TIME_dup(void *);
+const ASN1_ITEM *ASN1_ITEM_ptr(ASN1_ITEM_EXP *);
+
+/* These aren't macros these arguments are all const X on openssl > 1.0.x */
+
+int ASN1_TIME_print(BIO *, ASN1_TIME *);
+int ASN1_STRING_length(ASN1_STRING *);
+ASN1_STRING *ASN1_STRING_dup(ASN1_STRING *);
+int ASN1_STRING_cmp(ASN1_STRING *, ASN1_STRING *);
+int ASN1_UTCTIME_print(BIO *, ASN1_UTCTIME *);
+
+ASN1_OCTET_STRING *ASN1_OCTET_STRING_dup(ASN1_OCTET_STRING *);
+int ASN1_OCTET_STRING_cmp(ASN1_OCTET_STRING *, ASN1_OCTET_STRING *);
+
+ASN1_INTEGER *ASN1_INTEGER_dup(ASN1_INTEGER *);
+int ASN1_INTEGER_cmp(ASN1_INTEGER *, ASN1_INTEGER *);
+long ASN1_INTEGER_get(ASN1_INTEGER *);
+
+BIGNUM *ASN1_INTEGER_to_BN(ASN1_INTEGER *, BIGNUM *);
+ASN1_INTEGER *BN_to_ASN1_INTEGER(BIGNUM *, ASN1_INTEGER *);
+
+/* These isn't a macro the arg is const on openssl 1.0.2+ */
+int ASN1_GENERALIZEDTIME_check(ASN1_GENERALIZEDTIME *);
+int ASN1_UTCTIME_check(ASN1_UTCTIME *);
+
+/* Not a macro, const on openssl 1.0 */
+int ASN1_STRING_set_default_mask_asc(char *);
+
+int i2d_ASN1_TYPE(ASN1_TYPE *, unsigned char **);
+ASN1_TYPE *d2i_ASN1_TYPE(ASN1_TYPE **, const unsigned char **, long);
+"""
+
+CUSTOMIZATIONS = """
+/* This macro is removed in 1.1.0. We re-add it if required to support
+   pyOpenSSL versions older than whatever resolves
+   https://github.com/pyca/pyopenssl/issues/431 */
+#if !defined(M_ASN1_TIME_dup)
+#define M_ASN1_TIME_dup(a) (ASN1_TIME *)ASN1_STRING_dup((const ASN1_STRING *)a)
+#endif
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bignum.py
@@ -0,0 +1,88 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bn.h>
+"""
+
+TYPES = """
+typedef ... BN_CTX;
+typedef ... BIGNUM;
+typedef int... BN_ULONG;
+"""
+
+FUNCTIONS = """
+BIGNUM *BN_new(void);
+void BN_free(BIGNUM *);
+
+BN_CTX *BN_CTX_new(void);
+void BN_CTX_free(BN_CTX *);
+
+void BN_CTX_start(BN_CTX *);
+BIGNUM *BN_CTX_get(BN_CTX *);
+void BN_CTX_end(BN_CTX *);
+
+BIGNUM *BN_copy(BIGNUM *, const BIGNUM *);
+BIGNUM *BN_dup(const BIGNUM *);
+
+int BN_set_word(BIGNUM *, BN_ULONG);
+BN_ULONG BN_get_word(const BIGNUM *);
+
+const BIGNUM *BN_value_one(void);
+
+char *BN_bn2hex(const BIGNUM *);
+int BN_hex2bn(BIGNUM **, const char *);
+int BN_dec2bn(BIGNUM **, const char *);
+
+int BN_bn2bin(const BIGNUM *, unsigned char *);
+BIGNUM *BN_bin2bn(const unsigned char *, int, BIGNUM *);
+
+int BN_num_bits(const BIGNUM *);
+
+int BN_cmp(const BIGNUM *, const BIGNUM *);
+int BN_add(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_sub(BIGNUM *, const BIGNUM *, const BIGNUM *);
+int BN_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_sqr(BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_div(BIGNUM *, BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_nnmod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_add(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sub(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_mul(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_mod_sqr(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+int BN_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *, const BIGNUM *,
+               BN_CTX *);
+int BN_gcd(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+BIGNUM *BN_mod_inverse(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_set_bit(BIGNUM *, int);
+int BN_clear_bit(BIGNUM *, int);
+
+int BN_is_bit_set(const BIGNUM *, int);
+
+int BN_mask_bits(BIGNUM *, int);
+"""
+
+MACROS = """
+int BN_num_bytes(const BIGNUM *);
+
+int BN_zero(BIGNUM *);
+int BN_one(BIGNUM *);
+int BN_mod(BIGNUM *, const BIGNUM *, const BIGNUM *, BN_CTX *);
+
+int BN_lshift(BIGNUM *, const BIGNUM *, int);
+int BN_lshift1(BIGNUM *, BIGNUM *);
+
+int BN_rshift(BIGNUM *, BIGNUM *, int);
+int BN_rshift1(BIGNUM *, BIGNUM *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
@@ -0,0 +1,145 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <openssl/bio.h>
+"""
+
+TYPES = """
+typedef struct bio_st BIO;
+typedef void bio_info_cb(BIO *, int, const char *, int, long, long);
+typedef ... bio_st;
+typedef ... BIO_METHOD;
+typedef ... BUF_MEM;
+
+static const int BIO_TYPE_MEM;
+static const int BIO_TYPE_FILE;
+static const int BIO_TYPE_FD;
+static const int BIO_TYPE_SOCKET;
+static const int BIO_TYPE_CONNECT;
+static const int BIO_TYPE_ACCEPT;
+static const int BIO_TYPE_NULL;
+static const int BIO_CLOSE;
+static const int BIO_NOCLOSE;
+static const int BIO_TYPE_SOURCE_SINK;
+static const int BIO_CTRL_RESET;
+static const int BIO_CTRL_EOF;
+static const int BIO_CTRL_SET;
+static const int BIO_CTRL_SET_CLOSE;
+static const int BIO_CTRL_FLUSH;
+static const int BIO_CTRL_DUP;
+static const int BIO_CTRL_GET_CLOSE;
+static const int BIO_CTRL_INFO;
+static const int BIO_CTRL_GET;
+static const int BIO_CTRL_PENDING;
+static const int BIO_CTRL_WPENDING;
+static const int BIO_C_FILE_SEEK;
+static const int BIO_C_FILE_TELL;
+static const int BIO_TYPE_NONE;
+static const int BIO_TYPE_NBIO_TEST;
+static const int BIO_TYPE_BIO;
+static const int BIO_TYPE_DESCRIPTOR;
+static const int BIO_FLAGS_READ;
+static const int BIO_FLAGS_WRITE;
+static const int BIO_FLAGS_IO_SPECIAL;
+static const int BIO_FLAGS_RWS;
+static const int BIO_FLAGS_SHOULD_RETRY;
+static const int BIO_TYPE_NULL_FILTER;
+static const int BIO_TYPE_SSL;
+static const int BIO_TYPE_MD;
+static const int BIO_TYPE_BUFFER;
+static const int BIO_TYPE_CIPHER;
+static const int BIO_TYPE_BASE64;
+static const int BIO_TYPE_FILTER;
+"""
+
+FUNCTIONS = """
+int BIO_free(BIO *);
+void BIO_vfree(BIO *);
+void BIO_free_all(BIO *);
+BIO *BIO_push(BIO *, BIO *);
+BIO *BIO_pop(BIO *);
+BIO *BIO_next(BIO *);
+BIO *BIO_find_type(BIO *, int);
+BIO *BIO_new_file(const char *, const char *);
+BIO *BIO_new_fp(FILE *, int);
+BIO *BIO_new_fd(int, int);
+BIO *BIO_new_socket(int, int);
+long BIO_ctrl(BIO *, int, long, void *);
+long BIO_callback_ctrl(
+    BIO *,
+    int,
+    void (*)(struct bio_st *, int, const char *, int, long, long)
+);
+long BIO_int_ctrl(BIO *, int, long, int);
+size_t BIO_ctrl_pending(BIO *);
+size_t BIO_ctrl_wpending(BIO *);
+int BIO_read(BIO *, void *, int);
+int BIO_gets(BIO *, char *, int);
+int BIO_write(BIO *, const void *, int);
+int BIO_puts(BIO *, const char *);
+int BIO_method_type(const BIO *);
+
+int * Cryptography_bio_references(const BIO *);
+"""
+
+MACROS = """
+/* These added const to BIO_METHOD in 1.1.0 */
+BIO *BIO_new(BIO_METHOD *);
+BIO_METHOD *BIO_s_mem(void);
+BIO_METHOD *BIO_s_file(void);
+BIO_METHOD *BIO_s_fd(void);
+BIO_METHOD *BIO_s_socket(void);
+BIO_METHOD *BIO_s_null(void);
+BIO_METHOD *BIO_f_null(void);
+BIO_METHOD *BIO_f_buffer(void);
+/* BIO_new_mem_buf became const void * in 1.0.2g */
+BIO *BIO_new_mem_buf(void *, int);
+long BIO_set_fd(BIO *, long, int);
+long BIO_get_fd(BIO *, char *);

From pypy.commits at gmail.com  Thu Nov 17 08:32:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 05:32:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: remove _ssl module from the list in
 pypy options
Message-ID: <582db167.07452e0a.8034a.32eb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88438:0af0b1854309
Date: 2016-11-17 14:31 +0100
http://bitbucket.org/pypy/pypy/changeset/0af0b1854309/

Log:	remove _ssl module from the list in pypy options

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
-    "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
+    "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
@@ -118,7 +118,6 @@
     "zlib"      : ["rpython.rlib.rzlib"],
     "bz2"       : ["pypy.module.bz2.interp_bz2"],
     "pyexpat"   : ["pypy.module.pyexpat.interp_pyexpat"],
-    "_ssl"      : ["pypy.module._ssl.interp_ssl"],
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],

From pypy.commits at gmail.com  Thu Nov 17 09:32:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 06:32:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: hg merge py3.5
Message-ID: <582dbf83.973f1c0a.ce433.5c1a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88439:a915a719632d
Date: 2016-11-17 14:42 +0000
http://bitbucket.org/pypy/pypy/changeset/a915a719632d/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 16450 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls

From pypy.commits at gmail.com  Thu Nov 17 09:36:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 06:36:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: fix test
Message-ID: <582dc051.87a3190a.b9a92.37d5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88440:aed287295356
Date: 2016-11-17 14:45 +0000
http://bitbucket.org/pypy/pypy/changeset/aed287295356/

Log:	fix test

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -869,7 +869,7 @@
         with a: pass
         """
         code = compile_with_astcompiler(source, 'exec', self.space)
-        assert code.co_stacksize == 6  # i.e. <= 7, there is no systematic leak
+        assert code.co_stacksize == 5  # i.e. <= 7, there is no systematic leak
 
     def test_stackeffect_bug5(self):
         source = """if 1:

From pypy.commits at gmail.com  Thu Nov 17 09:55:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 06:55:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: Add a (passing)
 check for sys_exc_info being saved/restored in all cases
Message-ID: <582dc4cb.5043190a.885dc.3942@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88441:4bbf0a7c9f6c
Date: 2016-11-17 15:03 +0000
http://bitbucket.org/pypy/pypy/changeset/4bbf0a7c9f6c/

Log:	Add a (passing) check for sys_exc_info being saved/restored in all
	cases

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -248,8 +248,20 @@
         """Start this frame's execution."""
         if self._is_generator_or_coroutine():
             return self.initialize_as_generator(name, qualname)
+        elif we_are_translated():
+            return self.execute_frame()
         else:
-            return self.execute_frame()
+            # untranslated: check that sys_exc_info is exactly
+            # restored after running any Python function
+            executioncontext = self.space.getexecutioncontext()
+            exc_on_enter = executioncontext.sys_exc_info()
+            try:
+                w_res = self.execute_frame()
+            except OperationError:
+                assert exc_on_enter is executioncontext.sys_exc_info()
+                raise
+            assert exc_on_enter is executioncontext.sys_exc_info()
+            return w_res
     run._always_inline_ = True
 
     def initialize_as_generator(self, name, qualname):

From pypy.commits at gmail.com  Thu Nov 17 09:55:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 06:55:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: Before running a
 test, make sure sys_exc_info is cleared
Message-ID: <582dc4cd.521d190a.65478.3769@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88442:0418a55cd4e9
Date: 2016-11-17 15:03 +0000
http://bitbucket.org/pypy/pypy/changeset/0418a55cd4e9/

Log:	Before running a test, make sure sys_exc_info is cleared (might be a
	left-over from a previous failing test)

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -212,6 +212,7 @@
 
     def execute_appex(self, space, target, *args):
         self.space = space
+        space.getexecutioncontext().set_sys_exc_info(None)
         try:
             target(*args)
         except OperationError as e:

From pypy.commits at gmail.com  Thu Nov 17 10:20:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:20:20 -0800 (PST)
Subject: [pypy-commit] pypy default: Test depth_first_search
Message-ID: <582dcab4.9e002e0a.8f1a9.400c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88443:198fd97491aa
Date: 2016-11-17 16:00 +0100
http://bitbucket.org/pypy/pypy/changeset/198fd97491aa/

Log:	Test depth_first_search

diff --git a/rpython/tool/algo/test/test_graphlib.py b/rpython/tool/algo/test/test_graphlib.py
--- a/rpython/tool/algo/test/test_graphlib.py
+++ b/rpython/tool/algo/test/test_graphlib.py
@@ -20,6 +20,22 @@
         'G': [],
         }
 
+    def test_depth_first_search(self):
+        # 'D' missing from the list of vertices
+        lst = depth_first_search('A', list('ABCEFG'), self.edges)
+        assert lst == [
+            ('start', 'A'),
+            ('start', 'B'),
+            ('start', 'E'),
+            ('start', 'C'),
+            ('start', 'F'),
+            ('stop', 'F'),
+            ('stop', 'C'),
+            ('stop', 'E'),
+            ('stop', 'B'),
+            ('stop', 'A'),
+        ]
+
     def test_strong_components(self):
         edges = self.edges
         saved = copy_edges(edges)

From pypy.commits at gmail.com  Thu Nov 17 10:20:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:20:22 -0800 (PST)
Subject: [pypy-commit] pypy default: un-recursivify depth_first_search()
Message-ID: <582dcab6.d5092e0a.5d9b0.3850@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88444:c49289d2da96
Date: 2016-11-17 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/c49289d2da96/

Log:	un-recursivify depth_first_search()

diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py
--- a/rpython/tool/algo/graphlib.py
+++ b/rpython/tool/algo/graphlib.py
@@ -25,18 +25,27 @@
     return edges
 
 def depth_first_search(root, vertices, edges):
-    seen = {}
+    seen = set([root])
     result = []
-    def visit(vertex):
-        result.append(('start', vertex))
-        seen[vertex] = True
-        for edge in edges[vertex]:
-            w = edge.target
-            if w in vertices and w not in seen:
-                visit(w)
-        result.append(('stop', vertex))
-    visit(root)
-    return result
+    stack = []
+    while True:
+        result.append(('start', root))
+        stack.append((root, iter(edges[root])))
+        while True:
+            vertex, iterator = stack[-1]
+            try:
+                edge = next(iterator)
+            except StopIteration:
+                stack.pop()
+                result.append(('stop', vertex))
+                if not stack:
+                    return result
+            else:
+                w = edge.target
+                if w in vertices and w not in seen:
+                    seen.add(w)
+                    root = w
+                    break
 
 def vertices_reachable_from(root, vertices, edges):
     for event, v in depth_first_search(root, vertices, edges):

From pypy.commits at gmail.com  Thu Nov 17 10:20:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:20:24 -0800 (PST)
Subject: [pypy-commit] pypy default: un-recursivify all_cycles()
Message-ID: <582dcab8.6114190a.fa384.38b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88445:924339af3d65
Date: 2016-11-17 16:16 +0100
http://bitbucket.org/pypy/pypy/changeset/924339af3d65/

Log:	un-recursivify all_cycles()

diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py
--- a/rpython/tool/algo/graphlib.py
+++ b/rpython/tool/algo/graphlib.py
@@ -106,13 +106,20 @@
             for edge in edges[v]:
                 if edge.target in vertices:
                     edgestack.append(edge)
-                    visit(edge.target)
+                    yield visit(edge.target)
                     edgestack.pop()
             stackpos[v] = None
         else:
             if stackpos[v] is not None:   # back-edge
                 result.append(edgestack[stackpos[v]:])
-    visit(root)
+
+    pending = [visit(root)]
+    while pending:
+        generator = pending[-1]
+        try:
+            pending.append(next(generator))
+        except StopIteration:
+            pending.pop()
     return result        
 
 

From pypy.commits at gmail.com  Thu Nov 17 10:20:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:20:26 -0800 (PST)
Subject: [pypy-commit] pypy default: un-recursivify is_acyclic()
Message-ID: <582dcaba.ca26190a.e013f.37f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88446:da7ce96af92c
Date: 2016-11-17 16:19 +0100
http://bitbucket.org/pypy/pypy/changeset/da7ce96af92c/

Log:	un-recursivify is_acyclic()

diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py
--- a/rpython/tool/algo/graphlib.py
+++ b/rpython/tool/algo/graphlib.py
@@ -180,14 +180,20 @@
                 raise CycleFound
             if w in unvisited:
                 del unvisited[w]
-                visit(w)
+                yield visit(w)
         del visiting[vertex]
     try:
         unvisited = vertices.copy()
         while unvisited:
             visiting = {}
             root = unvisited.popitem()[0]
-            visit(root)
+            pending = [visit(root)]
+            while pending:
+                generator = pending[-1]
+                try:
+                    pending.append(next(generator))
+                except StopIteration:
+                    pending.pop()
     except CycleFound:
         return False
     else:

From pypy.commits at gmail.com  Thu Nov 17 10:36:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:36:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and fix: math.logN(very_large_int)
Message-ID: <582dce69.8721190a.e153c.3969@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88447:5b3bd2f5c4ef
Date: 2016-11-17 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/5b3bd2f5c4ef/

Log:	test and fix: math.logN(very_large_int)

diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -191,7 +191,9 @@
     try:
         try:
             x = _get_double(space, w_x)
-        except OverflowError:
+        except OperationError as e:
+            if not e.match(space, space.w_OverflowError):
+                raise
             if not space.isinstance_w(w_x, space.w_int):
                 raise
             # special case to support log(extremely-large-long)
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -167,6 +167,10 @@
         self.ftest(math.log10(100), 2)
         self.ftest(math.log10(0.01), -2)
 
+    def test_log_largevalue(self):
+        import math
+        assert math.log2(2**1234) == 1234.0
+
     def test_acosh(self):
         import math
         self.ftest(math.acosh(1), 0)

From pypy.commits at gmail.com  Thu Nov 17 10:36:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 07:36:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: dead code, left over from a merge
Message-ID: <582dce6b.50202e0a.fd1a8.3952@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88448:311ce573fdad
Date: 2016-11-17 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/311ce573fdad/

Log:	dead code, left over from a merge

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -324,8 +324,6 @@
                 raise
             except Exception as e:      # general fall-back
                 raise self._convert_unexpected_exception(e)
-                w_exitvalue = self.dispatch(self.pycode, next_instr,
-                                            executioncontext)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
             # it used to say self.last_exception = None

From pypy.commits at gmail.com  Thu Nov 17 11:04:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 08:04:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Seems at least this internal exception
 has both __cause__ and
Message-ID: <582dd4f1.22002e0a.9301c.3e17@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88449:a5e66da0d50f
Date: 2016-11-17 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/a5e66da0d50f/

Log:	Seems at least this internal exception has both __cause__ and
	__context__ attributes set

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -226,6 +226,7 @@
                                 space.wrap("%s raised StopIteration" %
                                            self.KIND))
             e2.chain_exceptions(space, e)
+            e2.set_cause(space, e.get_w_value(space))
             e2.record_context(space, self.frame)
             raise e2
         else:
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -775,3 +775,17 @@
 """, d)
         f = d['f']
         raises(RuntimeError, next, f(5))
+
+    def test_generator_stop_cause(self):
+        d = {}
+        exec("""from __future__ import generator_stop
+
+def gen1():
+    yield 42
+""", d)
+        my_gen = d['gen1']()
+        assert next(my_gen) == 42
+        stop_exc = StopIteration('spam')
+        e = raises(RuntimeError, my_gen.throw, StopIteration, stop_exc, None)
+        assert e.value.__cause__ is stop_exc
+        assert e.value.__context__ is stop_exc

From pypy.commits at gmail.com  Thu Nov 17 11:12:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 08:12:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: make new parameter optional to fix
 compression buildbot step
Message-ID: <582dd6d5.99012e0a.c53e5.3b89@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88450:04030a587cc7
Date: 2016-11-17 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/04030a587cc7/

Log:	make new parameter optional to fix compression buildbot step

diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -21,7 +21,7 @@
     "xx": None,    # for testing: 'None' should be completely ignored
     }
 
-def create_cffi_import_libraries(pypy_c, options, basedir, only):
+def create_cffi_import_libraries(pypy_c, options, basedir, only=None):
     from rpython.tool.runsubprocess import run_subprocess
 
     shutil.rmtree(str(join(basedir,'lib_pypy','__pycache__')),

From pypy.commits at gmail.com  Thu Nov 17 11:15:55 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Thu, 17 Nov 2016 08:15:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: force parameters of
 create_cffi_import_libraries to str paths
Message-ID: <582dd7bb.a516190a.1a141.3a64@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88451:4492eb10be95
Date: 2016-11-17 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/4492eb10be95/

Log:	force parameters of create_cffi_import_libraries to str paths

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -84,7 +84,7 @@
     if not _fake and not pypy_runs(pypy_c):
         raise OSError("Running %r failed!" % (str(pypy_c),))
     if not options.no_cffi:
-        failures = create_cffi_import_libraries(pypy_c, options, basedir)
+        failures = create_cffi_import_libraries(str(pypy_c), options, str(basedir))
         for key, module in failures:
             print >>sys.stderr, """!!!!!!!!!!\nBuilding {0} bindings failed.
                 You can either install development headers package,

From pypy.commits at gmail.com  Thu Nov 17 13:48:02 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Nov 2016 10:48:02 -0800 (PST)
Subject: [pypy-commit] pypy default: fix _ssl test_ztranslation
Message-ID: <582dfb62.42452e0a.f2fec.46c2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88452:5625ca7b74d8
Date: 2016-11-17 18:47 +0000
http://bitbucket.org/pypy/pypy/changeset/5625ca7b74d8/

Log:	fix _ssl test_ztranslation

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -182,6 +182,12 @@
             return s
             """)
 
+    def teardown_method(self, method):
+        # pytest may keep some objects alive.
+        # So do some clean-up now without waiting for them to die
+        from ..interp_ssl import SOCKET_STORAGE
+        SOCKET_STORAGE._dict.clear()
+
     def test_connect(self):
         import socket, gc
         ss = socket.ssl(self.s)
@@ -484,7 +490,7 @@
             c = _socket.socket()
             c.connect(s.getsockname())
             c.setblocking(False)
-            
+
             c = ctx._wrap_socket(c, False)
             try:
                 exc = raises(_ssl.SSLWantReadError, c.do_handshake)

From pypy.commits at gmail.com  Thu Nov 17 14:08:35 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 11:08:35 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Typo in the test, and fix the test
Message-ID: <582e0033.8259190a.5975f.455c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88453:9a277fe1750d
Date: 2016-11-17 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/9a277fe1750d/

Log:	Typo in the test, and fix the test

diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.interpreter.typedef import GetSetProperty
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.objspace.std.sliceobject import unwrap_start_stop
 from rpython.rlib.debug import check_nonneg
 from rpython.rlib.objectmodel import specialize
 
@@ -48,7 +49,9 @@
         self.data = [None] * BLOCKLEN
 
 class Lock(object):
-    pass
+    """This is not a lock.  It is a marker to detect concurrent
+    modifications (including in the single-threaded case).
+    """
 
 # ------------------------------------------------------------
 
@@ -340,31 +343,16 @@
     def iter(self):
         return W_DequeIter(self)
 
-    def index(self, w_x, w_start=None, w_stop=None):
+    @unwrap_spec(w_start=WrappedDefault(0), w_stop=WrappedDefault(sys.maxint))
+    def index(self, w_x, w_start, w_stop):
         space = self.space
         w_iter = space.iter(self)
         _len = self.len
-        start = 0
-        stop = _len
         lock = self.getlock()
 
-        if w_start is not None:
-            start = space.int_w(w_start)
-            if start < 0:
-                start += _len
-            if start < 0:
-                start = 0
-            elif start > _len:
-                start = _len
+        start, stop = unwrap_start_stop(space, _len, w_start, w_stop)
 
-        if w_stop is not None:
-            stop = space.int_w(w_stop)
-            if stop < 0:
-                stop += _len
-            if 0 <= stop > _len:
-                stop = _len
-
-        for i in range(0, stop):
+        for i in range(0, min(_len, stop)):
             try:
                 w_obj = space.next(w_iter)
                 if i < start:
diff --git a/pypy/module/_collections/test/test_deque.py b/pypy/module/_collections/test/test_deque.py
--- a/pypy/module/_collections/test/test_deque.py
+++ b/pypy/module/_collections/test/test_deque.py
@@ -416,4 +416,4 @@
         import sys
         elements = 'ABCDEFGHI'
         d = deque([-2, -1, 0, 0, 1, 2])
-        assert a.index(0, -4*sys.maxsize, 4*sys.maxsize) == 2
+        assert d.index(0, -4*sys.maxsize, 4*sys.maxsize) == 2

From pypy.commits at gmail.com  Thu Nov 17 14:08:37 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 11:08:37 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Pass
 _multiprocessing/test/test_connection.py -k test_exception
Message-ID: <582e0035.8786190a.3ee1b.4677@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88454:8a4da5e45a8c
Date: 2016-11-17 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/8a4da5e45a8c/

Log:	Pass _multiprocessing/test/test_connection.py -k test_exception

diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -2,6 +2,7 @@
 from errno import EINTR
 
 from rpython.rlib import rpoll, rsocket
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import lltype, rffi
 
@@ -29,7 +30,10 @@
             w_builtins, '__import__', space.wrap("pickle"))
 
 def BufferTooShort(space, w_data):
-    w_BufferTooShort = space.fromcache(State).w_BufferTooShort
+    state = space.fromcache(State)
+    if not we_are_translated() and not hasattr(state, 'w_BufferTooShort'):
+        state.init(space)   # xxx for test/test_connection.py
+    w_BufferTooShort = state.w_BufferTooShort
     return OperationError(w_BufferTooShort, w_data)
 
 def w_handle(space, handle):

From pypy.commits at gmail.com  Thu Nov 17 14:08:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 11:08:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Allow the current behavior as correct too
 (see comments)
Message-ID: <582e0037.5298190a.19c44.457f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88455:d80ac9390f6f
Date: 2016-11-17 18:11 +0100
http://bitbucket.org/pypy/pypy/changeset/d80ac9390f6f/

Log:	Allow the current behavior as correct too (see comments)

diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -182,7 +182,22 @@
             b = self.array(t, b'\x00' * a.itemsize * 2)
             assert len(b) == 2 and b[0] == 0 and b[1] == 0
             if t in 'bB':
-                raises(BufferError, a.frombytes, a)
+                old_items = a.tolist()
+                try:
+                    a.frombytes(a)
+                except BufferError:
+                    # CPython behavior:
+                    # "cannot resize an array that is exporting buffers"
+                    # This is the general error we get when we try to
+                    # resize the array while a buffer to that array is
+                    # alive.
+                    assert a.tolist() == old_items
+                else:
+                    # PyPy behavior: we can't reasonably implement that.
+                    # It's harder to crash PyPy in this case, but not
+                    # impossible, because of get_raw_address().  Too
+                    # bad I suppose.
+                    assert a.tolist() == old_items * 2
             else:
                 raises(TypeError, a.frombytes, a)
 

From pypy.commits at gmail.com  Thu Nov 17 14:08:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 17 Nov 2016 11:08:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <582e0039.50cc190a.f9d72.479c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88456:b6bd9894b86e
Date: 2016-11-17 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/b6bd9894b86e/

Log:	Fix test

diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -395,7 +395,12 @@
                 source = case.format(keyword)
                 exc = raises(SyntaxError, exec_, source)
                 assert custom_msg in exc.value.msg
-                assert exc.value.args[0] == 'invalid syntax'
+                # XXX the following line passes on CPython but not on
+                # PyPy, but do we really care about this single special
+                # case?
+                #assert exc.value.args[0] == 'invalid syntax'
+
                 source = source.replace("foo", "(foo.)")
                 exc = raises(SyntaxError, exec_, source)
-                assert custom_msg not in exc.value.msg
+                assert (custom_msg not in exc.value.msg) == (
+                    ('print (' in source or 'exec (' in source))

From pypy.commits at gmail.com  Thu Nov 17 17:35:21 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 17 Nov 2016 14:35:21 -0800 (PST)
Subject: [pypy-commit] pypy default: check for NULL strides (prevent crash
 in cython tests)
Message-ID: <582e30a9.4c9d1c0a.641e6.0614@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88457:e782e19bf1de
Date: 2016-11-18 00:34 +0200
http://bitbucket.org/pypy/pypy/changeset/e782e19bf1de/

Log:	check for NULL strides (prevent crash in cython tests)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -386,7 +386,10 @@
         size = pybuf.c_len
         ndim = widen(pybuf.c_ndim)
         shape =   [pybuf.c_shape[i]   for i in range(ndim)]
-        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_strides:
+            strides = [pybuf.c_strides[i] for i in range(ndim)]
+        else:
+            strides = [1]
         if pybuf.c_format:
             format = rffi.charp2str(pybuf.c_format)
         else:

From pypy.commits at gmail.com  Thu Nov 17 20:51:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Nov 2016 17:51:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Implement
 http://bugs.python.org/issue17636
Message-ID: <582e5e9e.84c3190a.4cc1b.5e1d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88458:3dc6e1434de4
Date: 2016-11-18 01:50 +0000
http://bitbucket.org/pypy/pypy/changeset/3dc6e1434de4/

Log:	Implement http://bugs.python.org/issue17636

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1019,14 +1019,25 @@
     def IMPORT_FROM(self, nameindex, next_instr):
         w_name = self.getname_w(nameindex)
         w_module = self.peekvalue()
+        self.pushvalue(self.import_from(w_module, w_name))
+
+    def import_from(self, w_module, w_name):
+        space = self.space
         try:
-            w_obj = self.space.getattr(w_module, w_name)
+            return space.getattr(w_module, w_name)
         except OperationError as e:
-            if not e.match(self.space, self.space.w_AttributeError):
+            if not e.match(space, space.w_AttributeError):
                 raise
-            raise oefmt(self.space.w_ImportError,
-                        "cannot import name %R", w_name)
-        self.pushvalue(w_obj)
+            try:
+                w_pkgname = space.getattr(
+                    w_module, space.newunicode(u'__name__'))
+                w_fullname = space.newunicode(u'%s.%s' %
+                    (space.unicode_w(w_pkgname), space.unicode_w(w_name)))
+                return space.getitem(space.sys.get('modules'), w_fullname)
+            except OperationError:
+                raise oefmt(
+                    space.w_ImportError, "cannot import name %R", w_name)
+
 
     def YIELD_VALUE(self, oparg, next_instr):
         raise Yield
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -128,6 +128,10 @@
              a = '',
              b = '',
              c = '')
+    setuppkg('circular',
+             circ1="from . import circ2",
+             circ2="from . import circ1")
+
     p = setuppkg("encoded",
              # actually a line 2, setuppkg() sets up a line1
              line2 = "# encoding: iso-8859-1\n",
@@ -179,7 +183,7 @@
 
 class AppTestImport(BaseFSEncodeTest):
     spaceconfig = {
-        "usemodules": ['_md5', 'time', 'struct', '_pypyjson'],
+        "usemodules": ['_md5', 'time', 'struct'],
     }
 
     def setup_class(cls):
@@ -510,6 +514,9 @@
         check_absolute()
         raises(TypeError, check_relative)
 
+    def test_relative_circular(self):
+        import circular.circ1  # doesn't fail
+
     def test_import_function(self):
         # More tests for __import__
         import sys

From pypy.commits at gmail.com  Thu Nov 17 21:26:58 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 17 Nov 2016 18:26:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Ensure that binascii.crc_hqx() only
 returns values below 0xffff (cpython issue #23728)
Message-ID: <582e66f2.90202e0a.de81b.5d8f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88459:d897004ed662
Date: 2016-11-18 02:26 +0000
http://bitbucket.org/pypy/pypy/changeset/d897004ed662/

Log:	Ensure that binascii.crc_hqx() only returns values below 0xffff
	(cpython issue #23728)

diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py
--- a/pypy/module/binascii/interp_hqx.py
+++ b/pypy/module/binascii/interp_hqx.py
@@ -240,7 +240,7 @@
 @unwrap_spec(data='bufferstr', oldcrc=int)
 def crc_hqx(space, data, oldcrc):
     "Compute hqx CRC incrementally."
-    crc = oldcrc
+    crc = oldcrc & 0xffff
     for c in data:
         crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
     return space.wrap(crc)
diff --git a/pypy/module/binascii/test/test_binascii.py b/pypy/module/binascii/test/test_binascii.py
--- a/pypy/module/binascii/test/test_binascii.py
+++ b/pypy/module/binascii/test/test_binascii.py
@@ -314,6 +314,7 @@
     def test_crc_hqx(self):
         for input, initial, expected in [
             (b"", 0, 0),
+            (b'', 0x12345, 0x2345),
             (b"", 123, 123),
             (b"hello", 321, 28955),
             (b"world", 65535, 12911),

From pypy.commits at gmail.com  Fri Nov 18 08:25:47 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Nov 2016 05:25:47 -0800 (PST)
Subject: [pypy-commit] pypy default: remove bogus test,
 __buffer__ is not a valid python attribute for an app-level class
Message-ID: <582f015b.42472e0a.5db09.8095@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88461:e17b6c7aa0ba
Date: 2016-11-18 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/e17b6c7aa0ba/

Log:	remove bogus test, __buffer__ is not a valid python attribute for an
	app-level class

diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -139,13 +139,6 @@
             return result;
             ''')
         assert 'foo\0bar\0baz' == pybuffer(buffer('foo\0bar\0baz'))
-        import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            class A(object):
-                def __buffer__(self, flags):
-                    return buffer('123')
-            assert pybuffer(A()) == '123'
-
 
     def test_pyarg_parse_string_fails(self):
         """

From pypy.commits at gmail.com  Fri Nov 18 08:25:49 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Nov 2016 05:25:49 -0800 (PST)
Subject: [pypy-commit] pypy default: refactor PyType_Ready into more_init
 for py3.5 compatibiility
Message-ID: <582f015d.07452e0a.8034a.867d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88462:9a8f9cff8c5d
Date: 2016-11-18 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/9a8f9cff8c5d/

Log:	refactor PyType_Ready into more_init for py3.5 compatibiility

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -350,7 +350,6 @@
                 PyObject * obj;
                 char * data;
                 int len;
-                PyType_Ready(&PyStringArrType_Type);
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
@@ -434,6 +433,7 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
             ''')
 
         a = module.newsubstr('abc')
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py
--- a/pypy/module/cpyext/test/test_intobject.py
+++ b/pypy/module/cpyext/test/test_intobject.py
@@ -85,7 +85,6 @@
                 if (!PyArg_ParseTuple(args, "Oi", &name, &intval))
                     return NULL;
 
-                PyType_Ready(&Enum_Type);
                 enumObj = PyObject_New(EnumObject, &Enum_Type);
                 if (!enumObj) {
                     return NULL;
@@ -160,7 +159,8 @@
                 /*tp_new*/              0
             };
             """, more_init = '''
-            Enum_Type.tp_base = &PyInt_Type;
+                Enum_Type.tp_base = &PyInt_Type;
+                if (PyType_Ready(&Enum_Type) < 0) INITERROR;
             ''')
 
         a = module.newEnum("ULTIMATE_ANSWER", 42)
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -29,11 +29,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_length = mp_length;
-                tp_as_mapping.mp_subscript = mp_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
@@ -44,8 +39,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue = '''
             static PyObject *
             mp_subscript(PyObject *self, PyObject *key)
             {
@@ -61,6 +55,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_length = mp_length;
+                tp_as_mapping.mp_subscript = mp_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj["hi there"] == 42
@@ -80,11 +80,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_length = sq_length;
-                tp_as_sequence.sq_item = sq_item;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''),
@@ -94,8 +89,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue='''
             static PyObject *
             sq_item(PyObject *self, Py_ssize_t size)
             {
@@ -111,6 +105,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_length = sq_length;
+                tp_as_sequence.sq_item = sq_item;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj[1] == 42
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -233,9 +233,9 @@
         if cls.runappdirect:
             try:
                 import numpy
-                cls.w_numpy_include = [numpy.get_include()]
-            except:
+            except ImportError:
                 skip('numpy not importable')
+            cls.w_numpy_include = [numpy.get_include()]
         else:
             numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
@@ -306,9 +306,6 @@
                 ),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifdef PYPY_VERSION
@@ -351,9 +348,6 @@
                 '''),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 ''',
@@ -409,9 +403,6 @@
                 """),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifndef PYPY_VERSION
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -670,15 +670,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
@@ -693,6 +688,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
@@ -704,15 +704,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_contains = sq_contains;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             sq_contains(PyObject *self, PyObject *value)
             {
@@ -723,6 +718,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         res = "foo" in obj
@@ -863,11 +863,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type.tp_as_number = &intlike_as_number;
-                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
-                intlike_as_number.nb_add = intlike_nb_add;
-                intlike_as_number.nb_power = intlike_nb_pow;
-                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
                 intObj = PyObject_New(IntLikeObject, &IntLike_Type);
                 if (!intObj) {
                     return NULL;
@@ -884,8 +879,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
-                if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL;
                 intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp);
                 if (!intObjNoOp) {
                     return NULL;
@@ -893,8 +886,7 @@
 
                 intObjNoOp->ival = intval;
                 return (PyObject *)intObjNoOp;
-             """)], prologue=
-            """
+             """)], prologue="""
             #include <math.h>
             typedef struct
             {
@@ -946,6 +938,14 @@
                 /*tp_name*/             "IntLikeNoOp",
                 /*tp_basicsize*/        sizeof(IntLikeObjectNoOp),
             };
+            """, more_init="""
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
+                intlike_as_number.nb_add = intlike_nb_add;
+                intlike_as_number.nb_power = intlike_nb_pow;
+                if (PyType_Ready(&IntLike_Type) < 0) INITERROR;
+                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
+                if (PyType_Ready(&IntLike_Type_NoOp) < 0) INITERROR;
             """)
         a = module.newInt(1)
         b = module.newInt(2)
@@ -993,19 +993,11 @@
         else:
             raise AssertionError("did not get TypeError!")
 
-    def test_call_tp_dealloc_when_created_from_python(self):
+    def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
             ("fetchFooType", "METH_VARARGS",
              """
                 PyObject *o;
-                Foo_Type.tp_basicsize = sizeof(FooObject);
-                Foo_Type.tp_dealloc = &dealloc_foo;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES
-                                    | Py_TPFLAGS_BASETYPE;
-                Foo_Type.tp_new = &new_foo;
-                Foo_Type.tp_free = &PyObject_Del;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
-
                 o = PyObject_New(PyObject, &Foo_Type);
                 init_foo(o);
                 Py_DECREF(o);   /* calls dealloc_foo immediately */
@@ -1024,8 +1016,7 @@
             ("getCounter", "METH_VARARGS",
              """
                 return PyInt_FromLong(foo_counter);
-             """)], prologue=
-            """
+             """)], prologue="""
             typedef struct {
                 PyObject_HEAD
                 int someval[99];
@@ -1059,6 +1050,14 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            """, more_init="""
+                Foo_Type.tp_basicsize = sizeof(FooObject);
+                Foo_Type.tp_dealloc = &dealloc_foo;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES
+                                    | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = &new_foo;
+                Foo_Type.tp_free = &PyObject_Del;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             """)
         Foo = module.fetchFooType()
         assert module.getCounter() == 1010
@@ -1100,14 +1099,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_call = &my_tp_call;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyObject *
             my_tp_call(PyObject *self, PyObject *args, PyObject *kwds)
             {
@@ -1117,6 +1112,10 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_call = &my_tp_call;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         x = module.new_obj()
         assert x() == 42
@@ -1126,18 +1125,18 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                FooType_Type.tp_base = &PyType_Type;
-                if (PyType_Ready(&FooType_Type) < 0) return NULL;
                 Py_INCREF(&FooType_Type);
                 return (PyObject *)&FooType_Type;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyTypeObject FooType_Type = {
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.Type",
             };
+            ''', more_init='''
+                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                FooType_Type.tp_base = &PyType_Type;
+                if (PyType_Ready(&FooType_Type) < 0) INITERROR;
             ''')
         FooType = module.getMetaClass()
         if not self.runappdirect:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -242,7 +242,7 @@
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
-    # test_call_tp_dealloc_when_created_from_python, we should not
+    # test_call_tp_dealloc, we should not
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -83,8 +83,11 @@
     def import_extension(self, modname, functions, prologue="",
             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
         body = prologue + make_methods(functions, modname)
-        init = """Py_InitModule("%s", methods);""" % (modname,)
+        init = """Py_InitModule("%s", methods);
+               """ % (modname,)
         if more_init:
+            init += """#define INITERROR return
+                    """
             init += more_init
         return self.import_module(
             name=modname, init=init, body=body, include_dirs=include_dirs,

From pypy.commits at gmail.com  Fri Nov 18 08:25:51 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Nov 2016 05:25:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-merge-PyType_Ready: merge relevant parts
 of 9a8f9cff8c5d
Message-ID: <582f015f.1a4b2e0a.d6bae.81b4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5-merge-PyType_Ready
Changeset: r88463:52e9d36465b2
Date: 2016-11-18 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/52e9d36465b2/

Log:	merge relevant parts of 9a8f9cff8c5d

diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -29,11 +29,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_length = mp_length;
-                tp_as_mapping.mp_subscript = mp_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
@@ -44,8 +39,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue = '''
             static PyObject *
             mp_subscript(PyObject *self, PyObject *key)
             {
@@ -61,6 +55,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_length = mp_length;
+                tp_as_mapping.mp_subscript = mp_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj["hi there"] == 42
@@ -76,11 +76,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_length = sq_length;
-                tp_as_sequence.sq_item = sq_item;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''),
@@ -90,8 +85,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue='''
             static PyObject *
             sq_item(PyObject *self, Py_ssize_t size)
             {
@@ -107,6 +101,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_length = sq_length;
+                tp_as_sequence.sq_item = sq_item;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj[1] == 42
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -235,9 +235,9 @@
         if cls.runappdirect:
             try:
                 import numpy
-                cls.w_numpy_include = [numpy.get_include()]
-            except:
+            except ImportError:
                 skip('numpy not importable')
+            cls.w_numpy_include = [numpy.get_include()]
         else:
             numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
@@ -308,9 +308,6 @@
                 ),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifdef PYPY_VERSION
@@ -353,9 +350,6 @@
                 '''),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 ''',
@@ -411,9 +405,6 @@
                 """),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifndef PYPY_VERSION
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -637,15 +637,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
@@ -660,6 +655,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
@@ -671,14 +671,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_contains = sq_contains;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             sq_contains(PyObject *self, PyObject *value)
             {
@@ -689,6 +685,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         res = "foo" in obj
@@ -826,11 +827,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type.tp_as_number = &intlike_as_number;
-                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
-                intlike_as_number.nb_add = intlike_nb_add;
-                intlike_as_number.nb_power = intlike_nb_pow;
-                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
                 intObj = PyObject_New(IntLikeObject, &IntLike_Type);
                 if (!intObj) {
                     return NULL;
@@ -847,8 +843,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT;
-                if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL;
                 intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp);
                 if (!intObjNoOp) {
                     return NULL;
@@ -856,8 +850,7 @@
 
                 intObjNoOp->ival = intval;
                 return (PyObject *)intObjNoOp;
-             """)], prologue=
-            """
+             """)], prologue="""
             #include <math.h>
             typedef struct
             {
@@ -909,6 +902,14 @@
                 /*tp_name*/             "IntLikeNoOp",
                 /*tp_basicsize*/        sizeof(IntLikeObjectNoOp),
             };
+            """, more_init="""
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
+                intlike_as_number.nb_add = intlike_nb_add;
+                intlike_as_number.nb_power = intlike_nb_pow;
+                if (PyType_Ready(&IntLike_Type) < 0) INITERROR;
+                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT;
+                if (PyType_Ready(&IntLike_Type_NoOp) < 0) INITERROR;
             """)
         a = module.newInt(1)
         b = module.newInt(2)
@@ -955,18 +956,11 @@
         else:
             raise AssertionError("did not get TypeError!")
 
-    def test_call_tp_dealloc_when_created_from_python(self):
+    def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
             ("fetchFooType", "METH_VARARGS",
              """
                 PyObject *o;
-                Foo_Type.tp_basicsize = sizeof(FooObject);
-                Foo_Type.tp_dealloc = &dealloc_foo;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
-                Foo_Type.tp_new = &new_foo;
-                Foo_Type.tp_free = &PyObject_Del;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
-
                 o = PyObject_New(PyObject, &Foo_Type);
                 init_foo(o);
                 Py_DECREF(o);   /* calls dealloc_foo immediately */
@@ -1020,6 +1014,13 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            """, more_init="""
+                Foo_Type.tp_basicsize = sizeof(FooObject);
+                Foo_Type.tp_dealloc = &dealloc_foo;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = &new_foo;
+                Foo_Type.tp_free = &PyObject_Del;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             """)
         Foo = module.fetchFooType()
         assert module.getCounter() == 1010
@@ -1061,14 +1062,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_call = &my_tp_call;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyObject *
             my_tp_call(PyObject *self, PyObject *args, PyObject *kwds)
             {
@@ -1078,6 +1075,10 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_call = &my_tp_call;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         x = module.new_obj()
         assert x() == 42
@@ -1087,18 +1088,18 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                FooType_Type.tp_base = &PyType_Type;
-                if (PyType_Ready(&FooType_Type) < 0) return NULL;
                 Py_INCREF(&FooType_Type);
                 return (PyObject *)&FooType_Type;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyTypeObject FooType_Type = {
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.Type",
             };
+            ''', more_init='''
+                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                FooType_Type.tp_base = &PyType_Type;
+                if (PyType_Ready(&FooType_Type) < 0) INITERROR;
             ''')
         FooType = module.getMetaClass()
         if not self.runappdirect:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -241,7 +241,7 @@
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
-    # test_call_tp_dealloc_when_created_from_python, we should not
+    # test_call_tp_dealloc, we should not
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -86,8 +86,11 @@
     def import_extension(self, modname, functions, prologue="",
             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
         body = prologue + make_methods(functions, modname)
-        init = """PyObject *mod = PyModule_Create(&moduledef);"""
+        init = """PyObject *mod = PyModule_Create(&moduledef);
+               """
         if more_init:
+            init += """#define INITERROR return NULL
+                    """
             init += more_init
         init += "\nreturn mod;"
         return self.import_module(

From pypy.commits at gmail.com  Fri Nov 18 08:25:45 2016
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 Nov 2016 05:25:45 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <582f0159.024e2e0a.bda6c.8437@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88460:502ffda2620c
Date: 2016-11-18 08:01 +0200
http://bitbucket.org/pypy/pypy/changeset/502ffda2620c/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -14,3 +14,9 @@
 .. branch: union-side-effects-2
 
 Try to improve the consistency of RPython annotation unions.
+
+.. branch: pytest-2.9.2
+
+.. branch: clean-exported-state
+
+Clean-ups in the jit optimizeopt
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -182,6 +182,12 @@
             return s
             """)
 
+    def teardown_method(self, method):
+        # pytest may keep some objects alive.
+        # So do some clean-up now without waiting for them to die
+        from ..interp_ssl import SOCKET_STORAGE
+        SOCKET_STORAGE._dict.clear()
+
     def test_connect(self):
         import socket, gc
         ss = socket.ssl(self.s)
@@ -484,7 +490,7 @@
             c = _socket.socket()
             c.connect(s.getsockname())
             c.setblocking(False)
-            
+
             c = ctx._wrap_socket(c, False)
             try:
                 exc = raises(_ssl.SSLWantReadError, c.do_handshake)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -387,7 +387,10 @@
         size = pybuf.c_len
         ndim = widen(pybuf.c_ndim)
         shape =   [pybuf.c_shape[i]   for i in range(ndim)]
-        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_strides:
+            strides = [pybuf.c_strides[i] for i in range(ndim)]
+        else:
+            strides = [1]
         if pybuf.c_format:
             format = rffi.charp2str(pybuf.c_format)
         else:
diff --git a/pytest.py b/pytest.py
--- a/pytest.py
+++ b/pytest.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # PYTHON_ARGCOMPLETE_OK
 """
 pytest: unit and functional testing with Python.
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -108,7 +108,7 @@
     """
     log_noopt = False
 
-    def __init__(self, trace, celltoken, state,
+    def __init__(self, trace, celltoken, state, runtime_boxes,
                  call_pure_results=None, enable_opts=None,
                  inline_short_preamble=True):
         self.trace = trace
@@ -117,6 +117,8 @@
         self.state = state
         self.call_pure_results = call_pure_results
         self.inline_short_preamble = inline_short_preamble
+        assert runtime_boxes is not None
+        self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -124,7 +126,11 @@
         assert unroll # we should not be here if it's disabled
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.call_pure_results, self.inline_short_preamble)
+            self.runtime_boxes, self.call_pure_results, self.inline_short_preamble)
+
+    def forget_optimization_info(self):
+        self.state.forget_optimization_info()
+        CompileData.forget_optimization_info(self)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
@@ -290,7 +296,7 @@
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     jitcell_token.target_tokens = [start_descr]
-    loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -360,7 +366,7 @@
     history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
     enable_opts = jitdriver_sd.warmstate.enable_opts
     call_pure_results = metainterp.call_pure_results
-    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -372,6 +378,7 @@
         history.cut(cut)
         history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
         loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+                                     jumpargs,
                                      call_pure_results=call_pure_results,
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
@@ -516,7 +523,7 @@
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
-        #     somehoe for ContinueRunningNormally
+        #     somehow for ContinueRunningNormally
         if reset_values:
             item.reset_value()
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
      AbstractResOp, GuardResOp
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, we_are_debug
 from rpython.jit.metainterp.optimizeopt import info
         
 
@@ -172,7 +172,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getfield(descr, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.AbstractStructPtrInfo):
                 assert opinfo in self.cached_infos
         if isinstance(res, PreambleOp):
@@ -202,7 +202,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getitem(descr, self.index, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.ArrayPtrInfo):
                 assert opinfo in self.cached_infos
         if (isinstance(res, PreambleOp) and
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,9 +24,20 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
+def check_no_forwarding(lsts):
+    for lst in lsts:
+        for op in lst:
+            assert op.get_forwarded() is None
+
 class LoopInfo(object):
     label_op = None
 
+    def _check_no_forwarding(self):
+        pass
+
+    def forget_optimization_info(self):
+        pass
+
 class BasicLoopInfo(LoopInfo):
     def __init__(self, inputargs, quasi_immutable_deps, jump_op):
         self.inputargs = inputargs
@@ -555,7 +566,8 @@
         return (BasicLoopInfo(trace.inputargs, self.quasi_immutable_deps, last_op),
                 self._newoperations)
 
-    def _clean_optimization_info(self, lst):
+    @staticmethod
+    def _clean_optimization_info(lst):
         for op in lst:
             if op.get_forwarded() is not None:
                 op.set_forwarded(None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -5,6 +5,7 @@
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
+from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 
 class PreambleOp(AbstractResOp):
@@ -18,7 +19,7 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     op = None
-    
+
     def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
@@ -51,7 +52,13 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
-    pass
+    res = None
+
+    def _check_no_forwarding(self):
+        assert self.res.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.res.clear_forwarded()
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -101,6 +108,14 @@
                                        descr=sop.getdescr())
         return ProducedShortOp(self, preamble_op)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.getfield_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.getfield_op.clear_forwarded()
+
     def __repr__(self):
         return "HeapOp(%r)" % (self.res,)
 
@@ -193,6 +208,16 @@
                 l.append(pop)
         return l
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        self.one._check_no_forwarding()
+        self.two._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.one.forget_optimization_info()
+        self.two.forget_optimization_info()
+
     def repr(self, memo):
         return "CompoundOp(%s, %s, %s)" % (self.res.repr(memo),
                                            self.one.repr(memo),
@@ -203,7 +228,7 @@
 
 class ProducedShortOp(AbstractProducedShortOp):
     invented_name = False
-    
+
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
@@ -215,6 +240,14 @@
     def repr(self, memo):
         return self.short_op.repr(memo)
 
+    def _check_no_forwarding(self):
+        self.short_op._check_no_forwarding()
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.short_op.forget_optimization_info()
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
 
@@ -235,6 +268,14 @@
     def repr(self, memo):
         return "INP(%s)" % (self.res.repr(memo),)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
@@ -454,16 +495,23 @@
         self.sb = sb
         self.extra_same_as = self.sb.extra_same_as
         self.target_token = target_token
+        self.build_inplace = False
 
     def setup(self, jump_args, short, label_args):
         self.jump_args = jump_args
         self.short = short
         self.label_args = label_args
+        self.build_inplace = True
 
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
         """
+        # Could this be considered a speculative error?
+        # This check should only fail when trying to jump to an existing trace
+        # by forcing portions of the virtualstate.
+        if not self.build_inplace:
+            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         op = preamble_op.op.get_box_replacement()
         if preamble_op.invented_name:
             self.extra_same_as.append(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -573,6 +573,7 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
+        state[0]._check_no_forwarding()
         return state
 
     def _convert_call_pure_results(self, d):
@@ -621,7 +622,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
         loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, call_pure_results)
+            celltoken, start_state, runtime_boxes, call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimize import InvalidLoop, SpeculativeError
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo
+     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo, check_no_forwarding
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
@@ -35,7 +35,7 @@
 
     def setinfo_from_preamble_list(self, lst, infos):
         for item in lst:
-            if item is None:
+            if item is None or isinstance(item, Const):
                 continue
             i = infos.get(item, None)
             if i is not None:
@@ -97,7 +97,6 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
-
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
@@ -115,26 +114,22 @@
         return modifier.get_virtual_state(args)
 
     def _check_no_forwarding(self, lsts, check_newops=True):
-        for lst in lsts:
-            for op in lst:
-                assert op.get_forwarded() is None
+        check_no_forwarding(lsts)
         if check_newops:
             assert not self.optimizer._newoperations
 
-
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs,
-                                           runtime_boxes, memo)
+                                           info.inputargs, memo)
         exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state,
+    def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
                              call_pure_results, inline_short_preamble=True):
         trace = trace.get_iter()
         try:
@@ -186,7 +181,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
+                    end_jump, label_op, runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -199,7 +194,7 @@
             # to the preamble.
             try:
                 new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+                        end_jump, label_op, runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
@@ -276,8 +271,7 @@
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, runtime_boxes,
-                                           box_names_memo)
+                                           info.inputargs, box_names_memo)
         exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
@@ -440,8 +434,7 @@
                 continue
             self._expand_info(item, infos)
 
-    def export_state(self, original_label_args, renamed_inputargs,
-                     runtime_boxes, memo):
+    def export_state(self, original_label_args, renamed_inputargs, memo):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
         self.optimizer.flush()
@@ -462,16 +455,17 @@
             op = produced_op.short_op.res
             if not isinstance(op, Const):
                 self._expand_info(op, infos)
-        self.optimizer._clean_optimization_info(end_args)
         return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
-                             short_inputargs, runtime_boxes, memo)
+                             short_inputargs, memo)
 
     def import_state(self, targetargs, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
                 len(targetargs))
+        self._check_no_forwarding([targetargs])
+        exported_state._check_no_forwarding()
         for i, target in enumerate(exported_state.next_iteration_args):
             source = targetargs[i]
             assert source is not target
@@ -527,13 +521,11 @@
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
     * quasi_immutable_deps - for tracking quasi immutables
-    * runtime_boxes - runtime values for boxes, necessary when generating
-                      guards to jump to
     """
 
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
-                 short_inputargs, runtime_boxes, memo):
+                 short_inputargs, memo):
         self.end_args = end_args
         self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
@@ -541,8 +533,8 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
-        self.runtime_boxes = runtime_boxes
         self.dump(memo)
+        self.forget_optimization_info()
 
     def dump(self, memo):
         if have_debug_prints():
@@ -552,5 +544,35 @@
                 debug_print("  " + box.repr(memo))
             debug_stop("jit-log-exported-state")
 
+    def _check_no_forwarding(self):
+        """ Ensures that no optimization state is attached to relevant operations
+        before importing anything. """
+        # Some of these may be redunant
+        check_no_forwarding([
+            self.end_args,
+            self.next_iteration_args,
+            self.renamed_inputargs,
+            self.short_inputargs,
+            self.exported_infos.keys()])
+        for box in self.short_boxes:
+            box._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        """ Clean up optimization info on all operations stored in the ExportedState.
+
+        This function needs to be called when exporting the optimizer state to
+        prevent leaking of optimization information between invocations of the
+        optimizer.
+
+        That includes cleaning up in the event that optimize_peeled_loop() fails
+        with an InvalidLoop exception, as optimize_peeled_loop() mutates the
+        contents of ExportedState.
+        """
+        Optimizer._clean_optimization_info(self.renamed_inputargs)
+        for box in self.exported_infos.iterkeys():
+            box.clear_forwarded()
+        for box in self.short_boxes:
+            box.forget_optimization_info()
+
     def final(self):
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1976,6 +1976,8 @@
         self.aborted_tracing_greenkey = None
 
     def retrace_needed(self, trace, exported_state):
+        if not we_are_translated():
+            exported_state._check_no_forwarding()
         self.partial_trace = trace
         self.retracing_from = self.potential_retrace_position
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -52,6 +52,10 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
+    def clear_forwarded(self):
+        if self.get_forwarded() is not None:
+            self.set_forwarded(None)
+
     @specialize.arg(1)
     def get_box_replacement(op, not_const=False):
         # Read the chain "op, op._forwarded, op._forwarded._forwarded..."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -295,10 +295,15 @@
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
 _translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
+_rpy_assert_value = CDefinedIntSymbolic('RPY_ASSERT_VALUE', default=1)
 
 def we_are_translated_to_c():
     return we_are_translated() and _translated_to_c
 
+def we_are_debug():
+    """ Returns True when not translated or translated with debugging enabled. """
+    return not we_are_translated() or (_translated_to_c and _rpy_assert_value)
+
 # ____________________________________________________________
 
 def instantiate(cls, nonmovable=False):
diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py
--- a/rpython/tool/algo/graphlib.py
+++ b/rpython/tool/algo/graphlib.py
@@ -25,18 +25,27 @@
     return edges
 
 def depth_first_search(root, vertices, edges):
-    seen = {}
+    seen = set([root])
     result = []
-    def visit(vertex):
-        result.append(('start', vertex))
-        seen[vertex] = True
-        for edge in edges[vertex]:
-            w = edge.target
-            if w in vertices and w not in seen:
-                visit(w)
-        result.append(('stop', vertex))
-    visit(root)
-    return result
+    stack = []
+    while True:
+        result.append(('start', root))
+        stack.append((root, iter(edges[root])))
+        while True:
+            vertex, iterator = stack[-1]
+            try:
+                edge = next(iterator)
+            except StopIteration:
+                stack.pop()
+                result.append(('stop', vertex))
+                if not stack:
+                    return result
+            else:
+                w = edge.target
+                if w in vertices and w not in seen:
+                    seen.add(w)
+                    root = w
+                    break
 
 def vertices_reachable_from(root, vertices, edges):
     for event, v in depth_first_search(root, vertices, edges):
@@ -97,13 +106,20 @@
             for edge in edges[v]:
                 if edge.target in vertices:
                     edgestack.append(edge)
-                    visit(edge.target)
+                    yield visit(edge.target)
                     edgestack.pop()
             stackpos[v] = None
         else:
             if stackpos[v] is not None:   # back-edge
                 result.append(edgestack[stackpos[v]:])
-    visit(root)
+
+    pending = [visit(root)]
+    while pending:
+        generator = pending[-1]
+        try:
+            pending.append(next(generator))
+        except StopIteration:
+            pending.pop()
     return result        
 
 
@@ -164,14 +180,20 @@
                 raise CycleFound
             if w in unvisited:
                 del unvisited[w]
-                visit(w)
+                yield visit(w)
         del visiting[vertex]
     try:
         unvisited = vertices.copy()
         while unvisited:
             visiting = {}
             root = unvisited.popitem()[0]
-            visit(root)
+            pending = [visit(root)]
+            while pending:
+                generator = pending[-1]
+                try:
+                    pending.append(next(generator))
+                except StopIteration:
+                    pending.pop()
     except CycleFound:
         return False
     else:
diff --git a/rpython/tool/algo/test/test_graphlib.py b/rpython/tool/algo/test/test_graphlib.py
--- a/rpython/tool/algo/test/test_graphlib.py
+++ b/rpython/tool/algo/test/test_graphlib.py
@@ -20,6 +20,22 @@
         'G': [],
         }
 
+    def test_depth_first_search(self):
+        # 'D' missing from the list of vertices
+        lst = depth_first_search('A', list('ABCEFG'), self.edges)
+        assert lst == [
+            ('start', 'A'),
+            ('start', 'B'),
+            ('start', 'E'),
+            ('start', 'C'),
+            ('start', 'F'),
+            ('stop', 'F'),
+            ('stop', 'C'),
+            ('stop', 'E'),
+            ('stop', 'B'),
+            ('stop', 'A'),
+        ]
+
     def test_strong_components(self):
         edges = self.edges
         saved = copy_edges(edges)
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -31,8 +31,10 @@
 RPY_EXTERN
 void RPyAssertFailed(const char* filename, long lineno,
                      const char* function, const char *msg);
+#  define RPY_ASSERT_VALUE 1
 #else
 #  define RPyAssert(x, msg)   /* nothing */
+#  define RPY_ASSERT_VALUE 0
 #endif
 
 RPY_EXTERN

From pypy.commits at gmail.com  Fri Nov 18 09:17:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 06:17:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-refactor-sys_exc_info: Close branch,
 ready to merge
Message-ID: <582f0d8f.17052e0a.75161.89c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-refactor-sys_exc_info
Changeset: r88464:7a088a818f31
Date: 2016-11-18 15:04 +0100
http://bitbucket.org/pypy/pypy/changeset/7a088a818f31/

Log:	Close branch, ready to merge


From pypy.commits at gmail.com  Fri Nov 18 09:17:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 06:17:53 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove py.test.importorskip(): it might
 skip (not fail) also if it fails
Message-ID: <582f0d91.42212e0a.17fd5.886b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88465:18e3e31a4c63
Date: 2016-11-18 15:17 +0100
http://bitbucket.org/pypy/pypy/changeset/18e3e31a4c63/

Log:	Remove py.test.importorskip(): it might skip (not fail) also if it
	fails to import pkg_resources to check the version we provide. IMHO
	that's not good at all. So don't trust it and write try: ... except
	ImportError: explicitly. The version number checking can be written
	explicitly too, but in this case it can be removed (it was for older
	Python < 2.7).

diff --git a/pypy/module/select/conftest.py b/pypy/module/select/conftest.py
deleted file mode 100644
--- a/pypy/module/select/conftest.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import py
-
-def pytest_collect_directory():
-    py.test.importorskip("ctypes")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/pypy/module/test_lib_pypy/ctypes_tests/support.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/support.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/support.py
@@ -2,8 +2,6 @@
 import sys
 import ctypes
 
-py.test.importorskip("ctypes", "1.0.2")
-
 try:
     import _rawffi
 except ImportError:
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
@@ -3,7 +3,10 @@
 import ctypes
 from _ctypes import function
 
-_rawffi = py.test.importorskip("_rawffi")
+try:
+    import _rawffi
+except ImportError:
+    py.test.skip("app-level test only for PyPy")
 
 class TestErrno:
 
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
--- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py
+++ b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
@@ -9,7 +9,9 @@
 
 def pytest_funcarg__child(request):
     try:
-        pexpect = pytest.importorskip('pexpect')
+        import pexpect
+    except ImportError:
+        pytest.skip("no pexpect module")
     except SyntaxError:
         pytest.skip('pexpect wont work on py3k')
     child = pexpect.spawn(sys.executable, ['-S'], timeout=10)

From pypy.commits at gmail.com  Fri Nov 18 09:20:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 06:20:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-refactor-sys_exc_info
Message-ID: <582f0e2f.e1002e0a.877ff.88ad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88466:bdac33b95aeb
Date: 2016-11-18 15:19 +0100
http://bitbucket.org/pypy/pypy/changeset/bdac33b95aeb/

Log:	hg merge py3.5-refactor-sys_exc_info

	In an except: or finally: block, move the current exception to the
	executioncontext, and restore it afterwards, like CPython. This
	might make the exception object escape a bit more, but it allows
	other operations to work without forcing any frame.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -622,8 +622,8 @@
 
     ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP_START: 1,
-    ops.WITH_CLEANUP_FINISH: -2,
+    ops.WITH_CLEANUP_START: 0,
+    ops.WITH_CLEANUP_FINISH: -1,
     ops.LOAD_BUILD_CLASS: 1,
     ops.POP_BLOCK: 0,
     ops.POP_EXCEPT: -1,
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -869,7 +869,7 @@
         with a: pass
         """
         code = compile_with_astcompiler(source, 'exec', self.space)
-        assert code.co_stacksize == 6  # i.e. <= 7, there is no systematic leak
+        assert code.co_stacksize == 5  # i.e. <= 7, there is no systematic leak
 
     def test_stackeffect_bug5(self):
         source = """if 1:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -333,13 +333,12 @@
             tb = tb.next
         self._application_traceback = tb
 
-    def record_context(self, space, frame):
-        """Record a __context__ for this exception if one exists,
-        searching from the current frame.
+    def record_context(self, space, ec):
+        """Record a __context__ for this exception if one exists.
         """
         if self._context_recorded:
             return
-        last = frame._exc_info_unroll(space)
+        last = ec.sys_exc_info()
         try:
             if last is not None:
                 self.chain_exceptions(space, last)
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -28,6 +28,10 @@
     def __init__(self, space):
         self.space = space
         self.topframeref = jit.vref_None
+        # this is exposed to app-level as 'sys.exc_info()'.  At any point in
+        # time it is the exception caught by the topmost 'except ... as e:'
+        # app-level block.
+        self.sys_exc_operror = None
         self.w_tracefunc = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
@@ -222,7 +226,6 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
-    @jit.dont_look_inside
     def sys_exc_info(self):
         """Implements sys.exc_info().
         Return an OperationError instance or None.
@@ -230,23 +233,10 @@
         # NOTE: the result is not the wrapped sys.exc_info() !!!
 
         """
-        return self.gettopframe()._exc_info_unroll(self.space)
+        return self.sys_exc_operror
 
     def set_sys_exc_info(self, operror):
-        frame = self.gettopframe_nohidden()
-        if frame:     # else, the exception goes nowhere and is lost
-            frame.last_exception = operror
-
-    def clear_sys_exc_info(self):
-        # Find the frame out of which sys_exc_info() would return its result,
-        # and hack this frame's last_exception to become the cleared
-        # OperationError (which is different from None!).
-        frame = self.gettopframe_nohidden()
-        while frame:
-            if frame.last_exception is not None:
-                frame.last_exception = get_cleared_operation_error(self.space)
-                break
-            frame = self.getnextframe_nohidden(frame)
+        self.sys_exc_operror = operror
 
     @jit.dont_look_inside
     def settrace(self, w_func):
@@ -356,7 +346,6 @@
                     event == 'c_exception'):
                 return
 
-            last_exception = frame.last_exception
             if event == 'leaveframe':
                 event = 'return'
 
@@ -372,7 +361,6 @@
                     raise
 
             finally:
-                frame.last_exception = last_exception
                 self.is_tracing -= 1
 
     def checksignals(self):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -22,6 +22,7 @@
         self._qualname = qualname   # may be null, use get_qualname()
         if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
             self.register_finalizer(self.space)
+        self.saved_operr = None
 
     def get_name(self):
         # 'name' is a byte string that is valid utf-8
@@ -45,42 +46,6 @@
                            self.get_qualname(),
                            unicode(addrstring)))
 
-    def descr__reduce__(self, space):
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod = space.getbuiltinmodule('_pickle_support')
-        mod = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get(self.KIND + '_new')
-        w = space.wrap
-        if self.frame:
-            w_frame = self.frame._reduce_state(space)
-        else:
-            w_frame = space.w_None
-
-        tup = [w_frame, w(self.running)]
-        return space.newtuple([new_inst, space.newtuple([]),
-                               space.newtuple(tup)])
-
-    def descr__setstate__(self, space, w_args):
-        from rpython.rlib.objectmodel import instantiate
-        args_w = space.unpackiterable(w_args)
-        w_framestate, w_running = args_w
-        if space.is_w(w_framestate, space.w_None):
-            self.frame = None
-            self.space = space
-            self.pycode = None
-            self._name = None
-            self._qualname = None
-        else:
-            frame = instantiate(space.FrameClass)   # XXX fish
-            frame.descr__setstate__(space, w_framestate)
-            if isinstance(self, GeneratorIterator):
-                GeneratorIterator.__init__(self, frame)
-            elif isinstance(self, Coroutine):
-                Coroutine.__init__(self, frame)
-            else:
-                assert False
-        self.running = self.space.is_true(w_running)
-
     def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator/coroutine,
 return next yielded value or raise StopIteration."""
@@ -136,6 +101,10 @@
         space = self.space
         if self.running:
             raise oefmt(space.w_ValueError, "%s already executing", self.KIND)
+        ec = space.getexecutioncontext()
+        current_exc_info = ec.sys_exc_info()
+        if self.saved_operr is not None:
+            ec.set_sys_exc_info(self.saved_operr)
         self.running = True
         try:
             w_result = frame.execute_frame(self, w_arg_or_err)
@@ -150,6 +119,8 @@
         finally:
             frame.f_backref = jit.vref_None
             self.running = False
+            self.saved_operr = ec.sys_exc_info()
+            ec.set_sys_exc_info(current_exc_info)
         return w_result
 
     def resume_execute_frame(self, frame, w_arg_or_err):
@@ -162,7 +133,7 @@
             try:
                 self.next_yield_from(frame, w_yf, w_arg_or_err)
             except OperationError as operr:
-                operr.record_context(space, frame)
+                operr.record_context(space, space.getexecutioncontext())
                 return frame.handle_generator_error(operr)
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
@@ -227,7 +198,7 @@
                                            self.KIND))
             e2.chain_exceptions(space, e)
             e2.set_cause(space, e.get_w_value(space))
-            e2.record_context(space, self.frame)
+            e2.record_context(space, space.getexecutioncontext())
             raise e2
         else:
             space.warn(space.wrap(u"generator '%s' raised StopIteration"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -21,7 +21,7 @@
 
 # Define some opcodes used
 for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY SETUP_WITH
-POP_BLOCK END_FINALLY'''.split():
+SETUP_ASYNC_WITH POP_BLOCK END_FINALLY'''.split():
     globals()[op] = stdlib_opcode.opmap[op]
 HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT
 
@@ -66,7 +66,6 @@
     f_generator_wref         = rweakref.dead_ref  # for generators/coroutines
     f_generator_nowref       = None               # (only one of the two attrs)
     last_instr               = -1
-    last_exception           = None
     f_backref                = jit.vref_None
     
     escaped                  = False  # see mark_as_escaped()
@@ -249,8 +248,20 @@
         """Start this frame's execution."""
         if self._is_generator_or_coroutine():
             return self.initialize_as_generator(name, qualname)
+        elif we_are_translated():
+            return self.execute_frame()
         else:
-            return self.execute_frame()
+            # untranslated: check that sys_exc_info is exactly
+            # restored after running any Python function
+            executioncontext = self.space.getexecutioncontext()
+            exc_on_enter = executioncontext.sys_exc_info()
+            try:
+                w_res = self.execute_frame()
+            except OperationError:
+                assert exc_on_enter is executioncontext.sys_exc_info()
+                raise
+            assert exc_on_enter is executioncontext.sys_exc_info()
+            return w_res
     run._always_inline_ = True
 
     def initialize_as_generator(self, name, qualname):
@@ -326,10 +337,6 @@
                 raise self._convert_unexpected_exception(e)
             finally:
                 executioncontext.return_trace(self, w_exitvalue)
-            # it used to say self.last_exception = None
-            # this is now done by the code in pypyjit module
-            # since we don't want to invalidate the virtualizable
-            # for no good reason
             got_exception = False
         finally:
             executioncontext.leave(self, w_exitvalue, got_exception)
@@ -458,128 +465,6 @@
                 self.space, arguments, keywords, keywords_w, w_star,
                 w_starstar, methodcall=methodcall)
 
-    @jit.dont_look_inside
-    def descr__reduce__(self, space):
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod    = space.getbuiltinmodule('_pickle_support')
-        mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('frame_new')
-        w_tup_state = self._reduce_state(space)
-        nt = space.newtuple
-        return nt([new_inst, nt([]), w_tup_state])
-
-    @jit.dont_look_inside
-    def _reduce_state(self, space):
-        from pypy.module._pickle_support import maker # helper fns
-        w = space.wrap
-        nt = space.newtuple
-
-        if self.get_w_f_trace() is None:
-            f_lineno = self.get_last_lineno()
-        else:
-            f_lineno = self.getorcreatedebug().f_lineno
-
-        nlocals = self.pycode.co_nlocals
-        values_w = self.locals_cells_stack_w
-        w_locals_cells_stack = maker.slp_into_tuple_with_nulls(space, values_w)
-
-        w_blockstack = nt([block._get_state_(space) for block in self.get_blocklist()])
-        if self.last_exception is None:
-            w_exc_value = space.w_None
-            w_tb = space.w_None
-        else:
-            w_exc_value = self.last_exception.get_w_value(space)
-            w_tb = w(self.last_exception.get_traceback())
-
-        d = self.getorcreatedebug()
-        tup_state = [
-            w(self.f_backref()),
-            w(self.get_builtin()),
-            w(self.pycode),
-            w_locals_cells_stack,
-            w_blockstack,
-            w_exc_value, # last_exception
-            w_tb,        #
-            self.get_w_globals(),
-            w(self.last_instr),
-            w(self.frame_finished_execution),
-            w(f_lineno),
-            space.w_None,           #XXX placeholder for f_locals
-
-            #f_restricted requires no additional data!
-            space.w_None,
-
-            w(d.instr_lb),
-            w(d.instr_ub),
-            w(d.instr_prev_plus_one),
-            w(self.valuestackdepth),
-            ]
-        return nt(tup_state)
-
-    @jit.dont_look_inside
-    def descr__setstate__(self, space, w_args):
-        from pypy.module._pickle_support import maker # helper fns
-        from pypy.interpreter.pycode import PyCode
-        from pypy.interpreter.module import Module
-        args_w = space.unpackiterable(w_args, 17)
-        w_f_back, w_builtin, w_pycode, w_locals_cells_stack, w_blockstack, w_exc_value, w_tb,\
-            w_globals, w_last_instr, w_finished, w_f_lineno, w_f_locals, \
-            w_f_trace, w_instr_lb, w_instr_ub, w_instr_prev_plus_one, w_stackdepth = args_w
-
-        new_frame = self
-        pycode = space.interp_w(PyCode, w_pycode)
-
-        values_w = maker.slp_from_tuple_with_nulls(space, w_locals_cells_stack)
-        nfreevars = len(pycode.co_freevars)
-        closure = None
-        if nfreevars:
-            base = pycode.co_nlocals + len(pycode.co_cellvars)
-            closure = values_w[base: base + nfreevars]
-
-        # do not use the instance's __init__ but the base's, because we set
-        # everything like cells from here
-        # XXX hack
-        from pypy.interpreter.function import Function
-        outer_func = Function(space, None, closure=closure,
-                             forcename="fake")
-        PyFrame.__init__(self, space, pycode, w_globals, outer_func)
-        f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True)
-        new_frame.f_backref = jit.non_virtual_ref(f_back)
-
-        if space.config.objspace.honor__builtins__:
-            new_frame.builtin = space.interp_w(Module, w_builtin)
-        else:
-            assert space.interp_w(Module, w_builtin) is space.builtin
-        new_frame.set_blocklist([unpickle_block(space, w_blk)
-                                 for w_blk in space.unpackiterable(w_blockstack)])
-        self.locals_cells_stack_w = values_w[:]
-        valuestackdepth = space.int_w(w_stackdepth)
-        if not self._check_stack_index(valuestackdepth):
-            raise oefmt(space.w_ValueError, "invalid stackdepth")
-        assert valuestackdepth >= 0
-        self.valuestackdepth = valuestackdepth
-        if space.is_w(w_exc_value, space.w_None):
-            new_frame.last_exception = None
-        else:
-            from pypy.interpreter.pytraceback import PyTraceback
-            tb = space.interp_w(PyTraceback, w_tb)
-            new_frame.last_exception = OperationError(space.type(w_exc_value),
-                                                      w_exc_value, tb
-                                                      )
-        new_frame.last_instr = space.int_w(w_last_instr)
-        new_frame.frame_finished_execution = space.is_true(w_finished)
-        d = new_frame.getorcreatedebug()
-        d.f_lineno = space.int_w(w_f_lineno)
-
-        if space.is_w(w_f_trace, space.w_None):
-            d.w_f_trace = None
-        else:
-            d.w_f_trace = w_f_trace
-
-        d.instr_lb = space.int_w(w_instr_lb)   #the three for tracing
-        d.instr_ub = space.int_w(w_instr_ub)
-        d.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one)
-
     def hide(self):
         return self.pycode.hidden_applevel
 
@@ -724,7 +609,7 @@
             return space.wrap(self.getorcreatedebug().f_lineno)
 
     def fset_f_lineno(self, space, w_new_lineno):
-        "Returns the line number of the instruction currently being executed."
+        "Change the line number of the instruction currently being executed."
         try:
             new_lineno = space.int_w(w_new_lineno)
         except OperationError:
@@ -763,48 +648,62 @@
                         "can't jump to 'except' line as there's no exception")
 
         # Don't jump into or out of a finally block.
-        f_lasti_setup_addr = -1
-        new_lasti_setup_addr = -1
-        blockstack = []
+        # Unlike CPython, we can't jump into or out of an except block
+        # either---there would be a mess with SysExcInfoRestorer.
+        f_lasti_handler_addr = -1
+        new_lasti_handler_addr = -1
+        blockstack = [-1]    # list of odd length:
+                             #   * addr of most recent outermost handler
+                             # [ * addr of start of outermost block
+                             #   * addr of most recent handler in that block 
+                             #       (last two items repeated) ]
         addr = 0
         while addr < len(code):
             op = ord(code[addr])
-            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
-                blockstack.append([addr, False])
+            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
+                      SETUP_ASYNC_WITH):
+                blockstack.append(addr)
+                blockstack.append(-1)
             elif op == POP_BLOCK:
-                setup_op = ord(code[blockstack[-1][0]])
-                if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
-                    blockstack[-1][1] = True
-                else:
-                    blockstack.pop()
-            elif op == END_FINALLY:
-                if len(blockstack) > 0:
-                    setup_op = ord(code[blockstack[-1][0]])
-                    if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
-                        blockstack.pop()
+                if len(blockstack) < 3:
+                    raise oefmt(space.w_SystemError,
+                                "blocks not properly nested in this bytecode")
+                blockstack.pop()
+                setup_op = ord(code[blockstack.pop()])
+                if setup_op != SETUP_LOOP:
+                    blockstack[-1] = addr
+            elif op == END_FINALLY:    # "async for" nests blocks
+                blockstack[-1] = -1    # strangely, careful here
 
             if addr == new_lasti or addr == self.last_instr:
-                for ii in range(len(blockstack)):
-                    setup_addr, in_finally = blockstack[~ii]
-                    if in_finally:
-                        if addr == new_lasti:
-                            new_lasti_setup_addr = setup_addr
-                        if addr == self.last_instr:
-                            f_lasti_setup_addr = setup_addr
-                        break
+                ii = len(blockstack) - 1
+                while ii > 0 and blockstack[ii] == -1:
+                    ii -= 2
+                assert ii >= 0
+                handler_addr = blockstack[ii]
+                if addr == new_lasti:
+                    new_lasti_handler_addr = handler_addr
+                if addr == self.last_instr:
+                    f_lasti_handler_addr = handler_addr
 
             if op >= HAVE_ARGUMENT:
                 addr += 3
             else:
                 addr += 1
 
-        assert len(blockstack) == 0
+        if len(blockstack) != 1:
+            raise oefmt(space.w_SystemError,
+                        "blocks not properly nested in this bytecode")
 
-        if new_lasti_setup_addr != f_lasti_setup_addr:
+        if new_lasti_handler_addr != f_lasti_handler_addr:
             raise oefmt(space.w_ValueError,
-                        "can't jump into or out of a 'finally' block %d -> %d",
-                        f_lasti_setup_addr, new_lasti_setup_addr)
+                        "can't jump into or out of an 'expect' or "
+                        "'finally' block (%d -> %d)",
+                        f_lasti_handler_addr, new_lasti_handler_addr)
 
+        # now we know we're not jumping into or out of a place which
+        # needs a SysExcInfoRestorer.  Check that we're not jumping
+        # *into* a block, but only (potentially) out of some blocks.
         if new_lasti < self.last_instr:
             min_addr = new_lasti
             max_addr = self.last_instr
@@ -812,42 +711,40 @@
             min_addr = self.last_instr
             max_addr = new_lasti
 
-        delta_iblock = min_delta_iblock = 0
+        delta_iblock = min_delta_iblock = 0    # see below for comment
         addr = min_addr
         while addr < max_addr:
             op = ord(code[addr])
 
-            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
+            if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
+                      SETUP_ASYNC_WITH):
                 delta_iblock += 1
             elif op == POP_BLOCK:
                 delta_iblock -= 1
                 if delta_iblock < min_delta_iblock:
                     min_delta_iblock = delta_iblock
 
-            if op >= stdlib_opcode.HAVE_ARGUMENT:
+            if op >= HAVE_ARGUMENT:
                 addr += 3
             else:
                 addr += 1
 
-        f_iblock = 0
-        block = self.lastblock
-        while block:
-            f_iblock += 1
-            block = block.previous
-        min_iblock = f_iblock + min_delta_iblock
+        # 'min_delta_iblock' is <= 0; its absolute value is the number of
+        # blocks we exit.  'go_iblock' is the delta number of blocks
+        # between the last_instr and the new_lasti, in this order.
         if new_lasti > self.last_instr:
-            new_iblock = f_iblock + delta_iblock
+            go_iblock = delta_iblock
         else:
-            new_iblock = f_iblock - delta_iblock
+            go_iblock = -delta_iblock
 
-        if new_iblock > min_iblock:
+        if go_iblock > min_delta_iblock:
             raise oefmt(space.w_ValueError,
                         "can't jump into the middle of a block")
+        assert go_iblock <= 0
 
-        while f_iblock > new_iblock:
+        for ii in range(-go_iblock):
             block = self.pop_block()
-            block.cleanup(self)
-            f_iblock -= 1
+            block.cleanupstack(self)
 
         self.getorcreatedebug().f_lineno = new_lineno
         self.last_instr = new_lasti
@@ -880,55 +777,11 @@
     def fdel_f_trace(self, space):
         self.getorcreatedebug().w_f_trace = None
 
-    def fget_f_exc_type(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return f.last_exception.w_type
-        return space.w_None
-
-    def fget_f_exc_value(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return f.last_exception.get_w_value(space)
-        return space.w_None
-
-    def fget_f_exc_traceback(self, space):
-        if self.last_exception is not None:
-            f = self.f_backref()
-            while f is not None and f.last_exception is None:
-                f = f.f_backref()
-            if f is not None:
-                return space.wrap(f.last_exception.get_traceback())
-        return space.w_None
-
     def fget_f_restricted(self, space):
         if space.config.objspace.honor__builtins__:
             return space.wrap(self.builtin is not space.builtin)
         return space.w_False
 
-    @jit.unroll_safe
-    @specialize.arg(2)
-    def _exc_info_unroll(self, space, for_hidden=False):
-        """Return the most recent OperationError being handled in the
-        call stack
-        """
-        frame = self
-        while frame:
-            last = frame.last_exception
-            if last is not None:
-                if last is get_cleared_operation_error(self.space):
-                    break
-                if for_hidden or not frame.hide():
-                    return last
-            frame = frame.f_backref()
-        return None
-
     def get_generator(self):
         if self.space.config.translation.rweakref:
             return self.f_generator_wref()
@@ -936,10 +789,9 @@
             return self.f_generator_nowref
 
     def descr_clear(self, space):
+        """F.clear(): clear most references held by the frame"""
         # Clears a random subset of the attributes (e.g. the fast
-        # locals, but not f_locals).  Also clears last_exception, which
-        # is not quite like CPython when it clears f_exc_* (however
-        # there might not be an observable difference).
+        # locals, but not f_locals).
         if not self.frame_finished_execution:
             if not self._is_generator_or_coroutine():
                 raise oefmt(space.w_RuntimeError,
@@ -953,7 +805,6 @@
                 # awaited" in this case too.  Does it make any sense?
                 gen.descr_close()
 
-        self.last_exception = None
         debug = self.getdebug()
         if debug is not None:
             debug.w_f_trace = None
@@ -967,6 +818,7 @@
                 w_newvalue = None
             self.locals_cells_stack_w[i] = w_newvalue
         self.valuestackdepth = 0
+        self.lastblock = None    # the FrameBlock chained list
 
     def _convert_unexpected_exception(self, e):
         from pypy.interpreter import error
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -63,17 +63,14 @@
         try:
             while True:
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
-        except Return:
-            self.last_exception = None
-            return self.popvalue()
-        except Yield:
+        except ExitFrame:
             return self.popvalue()
 
     def handle_bytecode(self, co_code, next_instr, ec):
         try:
             next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
         except OperationError as operr:
-            operr.record_context(self.space, self)
+            operr.record_context(self.space, ec)
             next_instr = self.handle_operation_error(ec, operr)
         except RaiseWithExplicitTraceback as e:
             next_instr = self.handle_operation_error(ec, e.operr,
@@ -688,12 +685,11 @@
         if nbargs > 2:
             raise BytecodeCorruption("bad RAISE_VARARGS oparg")
         if nbargs == 0:
-            last_operr = self._exc_info_unroll(space, for_hidden=True)
+            last_operr = self.space.getexecutioncontext().sys_exc_info()
             if last_operr is None:
                 raise oefmt(space.w_RuntimeError,
                             "No active exception to reraise")
             # re-raise, no new traceback obj will be attached
-            self.last_exception = last_operr
             raise RaiseWithExplicitTraceback(last_operr)
         if nbargs == 2:
             w_cause = self.popvalue()
@@ -748,12 +744,20 @@
 
     def POP_EXCEPT(self, oparg, next_instr):
         block = self.pop_block()
-        block.cleanup(self)
-        return
+        assert isinstance(block, SysExcInfoRestorer)
+        block.cleanupstack(self)   # restores ec.sys_exc_operror
 
     def POP_BLOCK(self, oparg, next_instr):
         block = self.pop_block()
-        block.cleanup(self)  # the block knows how to clean up the value stack
+        block.pop_block(self)  # the block knows how to clean up the value stack
+
+    def save_and_change_sys_exc_info(self, operationerr):
+        ec = self.space.getexecutioncontext()
+        last_exception = ec.sys_exc_info()
+        block = SysExcInfoRestorer(last_exception, self.lastblock)
+        self.lastblock = block
+        if operationerr is not None:   # otherwise, don't change sys_exc_info
+            ec.set_sys_exc_info(operationerr)
 
     def end_finally(self):
         # unlike CPython, there are two statically distinct cases: the
@@ -763,8 +767,14 @@
         #   [exception value we are now handling]
         #   [wrapped SApplicationException]
         # In the case of a finally: block, the stack contains only one
-        # item (unlike CPython which can have 1, 2 or 3 items):
+        # item (unlike CPython which can have 1, 2, 3 or 5 items, and
+        # even in one case a non-fixed number of items):
         #   [wrapped subclass of SuspendedUnroller]
+
+        block = self.pop_block()
+        assert isinstance(block, SysExcInfoRestorer)
+        block.cleanupstack(self)   # restores ec.sys_exc_operror
+
         w_top = self.popvalue()
         if self.space.is_w(w_top, self.space.w_None):
             # case of a finally: block with no exception
@@ -1148,8 +1158,8 @@
         w_exit = self.space.get(w_descr, w_manager)
         self.settopvalue(w_exit)
         w_result = self.space.get_and_call_function(w_enter, w_manager)
-        block = WithBlock(self.valuestackdepth,
-                          next_instr + offsettoend, self.lastblock)
+        block = FinallyBlock(self.valuestackdepth,
+                             next_instr + offsettoend, self.lastblock)
         self.lastblock = block
         self.pushvalue(w_result)
 
@@ -1161,39 +1171,28 @@
         if isinstance(w_unroller, SApplicationException):
             # app-level exception
             operr = w_unroller.operr
-            old_last_exception = self.last_exception
-            self.last_exception = operr
             w_traceback = self.space.wrap(operr.get_traceback())
-            try:
-                w_res = self.call_contextmanager_exit_function(
-                    w_exitfunc,
-                    operr.w_type,
-                    operr.get_w_value(self.space),
-                    w_traceback)
-            except:
-                self.last_exception = old_last_exception
-                raise
-            # push a marker that also contains the old_last_exception,
-            # which must be restored as 'self.last_exception' but only
-            # in WITH_CLEANUP_FINISH
-            self.pushvalue(SApplicationException(old_last_exception))
+            w_res = self.call_contextmanager_exit_function(
+                w_exitfunc,
+                operr.w_type,
+                operr.get_w_value(self.space),
+                w_traceback)
         else:
             w_res = self.call_contextmanager_exit_function(
                 w_exitfunc,
                 self.space.w_None,
                 self.space.w_None,
                 self.space.w_None)
-            self.pushvalue(self.space.w_None)
         self.pushvalue(w_res)
+        # in the stack now:  [w_res, w_unroller-or-w_None..]
 
     def WITH_CLEANUP_FINISH(self, oparg, next_instr):
         w_suppress = self.popvalue()
-        w_marker = self.popvalue()
-        if isinstance(w_marker, SApplicationException):
-            self.last_exception = w_marker.operr    # may be None
-            if self.space.is_true(w_suppress):
-                # __exit__() returned True -> Swallow the exception.
-                self.settopvalue(self.space.w_None)
+        if self.space.is_true(w_suppress):
+            # __exit__() returned True -> Swallow the exception.
+            self.settopvalue(self.space.w_None)
+        # this is always followed by END_FINALLY
+        # in the stack now: [w_unroller-or-w_None..]
 
     @jit.unroll_safe
     def call_function(self, oparg, w_starstar=None, has_vararg=False):
@@ -1474,8 +1473,8 @@
 
     def SETUP_ASYNC_WITH(self, offsettoend, next_instr):
         res = self.popvalue()
-        block = WithBlock(self.valuestackdepth,
-                          next_instr + offsettoend, self.lastblock)
+        block = FinallyBlock(self.valuestackdepth,
+                             next_instr + offsettoend, self.lastblock)
         self.lastblock = block
         self.pushvalue(res)
 
@@ -1557,12 +1556,15 @@
 
 ### ____________________________________________________________ ###
 
+class ExitFrame(Exception):
+    pass
 
-class Return(Exception):
+
+class Return(ExitFrame):
     """Raised when exiting a frame via a 'return' statement."""
 
 
-class Yield(Exception):
+class Yield(ExitFrame):
     """Raised when exiting a frame via a 'yield' statement."""
 
 
@@ -1654,7 +1656,7 @@
     def cleanupstack(self, frame):
         frame.dropvaluesuntil(self.valuestackdepth)
 
-    def cleanup(self, frame):
+    def pop_block(self, frame):
         "Clean up a frame when we normally exit the block."
         self.cleanupstack(frame)
 
@@ -1691,28 +1693,29 @@
             return r_uint(self.handlerposition)
 
 
-class ExceptHandlerBlock(FrameBlock):
+class SysExcInfoRestorer(FrameBlock):
     """
-    This is a special, implicit block type which is created when entering an
-    except handler. It does not belong to any opcode
+    This is a special, implicit block type which is created when entering a
+    finally or except handler. It does not belong to any opcode
     """
 
     _immutable_ = True
-    _opname = 'EXCEPT_HANDLER_BLOCK' # it's not associated to any opcode
+    _opname = 'SYS_EXC_INFO_RESTORER' # it's not associated to any opcode
     handling_mask = 0 # this block is never handled, only popped by POP_EXCEPT
 
+    def __init__(self, operr, previous):
+        self.operr = operr
+        self.previous = previous
+
+    def pop_block(self, frame):
+        assert False # never called
+
     def handle(self, frame, unroller):
         assert False # never called
 
     def cleanupstack(self, frame):
-        frame.dropvaluesuntil(self.valuestackdepth+1)
-        w_last_exception = frame.popvalue()
-        if not isinstance(w_last_exception, W_OperationError):
-            msg = "expected an OperationError, got %s" % (
-                frame.space.str_w(w_last_exception))
-            raise BytecodeCorruption(msg)
-        frame.last_exception = w_last_exception.operr
-        FrameBlock.cleanupstack(self, frame)
+        ec = frame.space.getexecutioncontext()
+        ec.set_sys_exc_info(self.operr)
 
 
 class ExceptBlock(FrameBlock):
@@ -1726,22 +1729,18 @@
         # push the exception to the value stack for inspection by the
         # exception handler (the code after the except:)
         self.cleanupstack(frame)
+        # the stack setup is slightly different than in CPython:
+        # instead of the traceback, we store the unroller object,
+        # wrapped.
         assert isinstance(unroller, SApplicationException)
         operationerr = unroller.operr
         operationerr.normalize_exception(frame.space)
-        # the stack setup is slightly different than in CPython:
-        # instead of the traceback, we store the unroller object,
-        # wrapped.
-        # this is popped by POP_EXCEPT, which is present only in py3k
-        w_last_exception = W_OperationError(frame.last_exception)
-        w_last_exception = frame.space.wrap(w_last_exception)
-        frame.pushvalue(w_last_exception)
-        block = ExceptHandlerBlock(self.valuestackdepth, 0, frame.lastblock)
-        frame.lastblock = block
         frame.pushvalue(frame.space.wrap(unroller))
         frame.pushvalue(operationerr.get_w_value(frame.space))
         frame.pushvalue(operationerr.w_type)
-        frame.last_exception = operationerr
+        # set the current value of sys_exc_info to operationerr,
+        # saving the old value in a custom type of FrameBlock
+        frame.save_and_change_sys_exc_info(operationerr)
         return r_uint(self.handlerposition)   # jump to the handler
 
 
@@ -1751,7 +1750,6 @@
     _immutable_ = True
     _opname = 'SETUP_FINALLY'
     handling_mask = -1     # handles every kind of SuspendedUnroller
-    restore_last_exception = True # set to False by WithBlock
 
     def handle(self, frame, unroller):
         # any abnormal reason for unrolling a finally: triggers the end of
@@ -1763,53 +1761,48 @@
             operationerr = unroller.operr
             operationerr.normalize_exception(frame.space)
         frame.pushvalue(frame.space.wrap(unroller))
-        if operationerr and self.restore_last_exception:
-            frame.last_exception = operationerr
+        # set the current value of sys_exc_info to operationerr,
+        # saving the old value in a custom type of FrameBlock
+        frame.save_and_change_sys_exc_info(operationerr)
         return r_uint(self.handlerposition)   # jump to the handler
 
+    def pop_block(self, frame):
+        self.cleanupstack(frame)
+        frame.save_and_change_sys_exc_info(None)
 
-class WithBlock(FinallyBlock):
 
-    _immutable_ = True
-    restore_last_exception = False
-
-    def handle(self, frame, unroller):
-        if isinstance(unroller, SApplicationException):
-            unroller.operr.normalize_exception(frame.space)
-        return FinallyBlock.handle(self, frame, unroller)
-
-block_classes = {'EXCEPT_HANDLER_BLOCK': ExceptHandlerBlock,
+block_classes = {'SYS_EXC_INFO_RESTORER': SysExcInfoRestorer,
                  'SETUP_LOOP': LoopBlock,
                  'SETUP_EXCEPT': ExceptBlock,
                  'SETUP_FINALLY': FinallyBlock,
-                 'SETUP_WITH': WithBlock,
+                 'SETUP_WITH': FinallyBlock,
                  }
 
 
-class W_OperationError(W_Root):
-    """
-    Tiny applevel wrapper around an OperationError.
-    """
-
-    def __init__(self, operr):
-        self.operr = operr
-
-    def descr_reduce(self, space):
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod = space.getbuiltinmodule('_pickle_support')
-        mod = space.interp_w(MixedModule, w_mod)
-        w_new_inst = mod.get('operationerror_new')
-        w_args = space.newtuple([])
-        operr = self.operr
-        if operr is None:
-            return space.newtuple([w_new_inst, w_args])
-        w_state = space.newtuple([operr.w_type, operr.get_w_value(space),
-                                  operr.get_traceback()])
-        return space.newtuple([w_new_inst, w_args, w_state])
-
-    def descr_setstate(self, space, w_state):
-        w_type, w_value, w_tb = space.fixedview(w_state, 3)
-        self.operr = OperationError(w_type, w_value, w_tb)
+##class W_OperationError(W_Root):
+##    """
+##    Tiny applevel wrapper around an OperationError.
+##    """
+##
+##    def __init__(self, operr):
+##        self.operr = operr
+##
+##    def descr_reduce(self, space):
+##        from pypy.interpreter.mixedmodule import MixedModule
+##        w_mod = space.getbuiltinmodule('_pickle_support')
+##        mod = space.interp_w(MixedModule, w_mod)
+##        w_new_inst = mod.get('operationerror_new')
+##        w_args = space.newtuple([])
+##        operr = self.operr
+##        if operr is None:
+##            return space.newtuple([w_new_inst, w_args])
+##        w_state = space.newtuple([operr.w_type, operr.get_w_value(space),
+##                                  operr.get_traceback()])
+##        return space.newtuple([w_new_inst, w_args, w_state])
+##
+##    def descr_setstate(self, space, w_state):
+##        w_type, w_value, w_tb = space.fixedview(w_state, 3)
+##        self.operr = OperationError(w_type, w_value, w_tb)
 
 def source_as_str(space, w_source, funcname, what, flags):
     """Return source code as str0 with adjusted compiler flags
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -2,9 +2,8 @@
 from pypy.interpreter import executioncontext
 from pypy.interpreter.error import OperationError
 
-class Finished(OperationError):
-    def __init__(self):
-        OperationError.__init__(self, "exception_class", "exception_value")
+class Finished(Exception):
+    pass
 
 
 class TestExecutionContext:
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -457,6 +457,46 @@
         assert closed == [True]
         """
 
+    def test_exc_info_in_generator(self):
+        import sys
+        def g():
+            try:
+                raise ValueError
+            except ValueError:
+                yield sys.exc_info()[0]
+                yield sys.exc_info()[0]
+        try:
+            raise IndexError
+        except IndexError:
+            gen = g()
+            assert sys.exc_info()[0] is IndexError
+            assert next(gen) is ValueError
+            assert sys.exc_info()[0] is IndexError
+            assert next(gen) is ValueError
+            assert sys.exc_info()[0] is IndexError
+            raises(StopIteration, next, gen)
+            assert sys.exc_info()[0] is IndexError
+
+    def test_exc_info_in_generator_2(self):
+        import sys
+        def g():
+            yield sys.exc_info()[0]
+            try:
+                raise LookupError
+            except LookupError:
+                yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+        try:
+            raise IndexError
+        except IndexError:
+            gen = g()     # the IndexError is not captured at all
+        try:
+            raise ValueError
+        except ValueError:
+            assert next(gen) is ValueError
+            assert next(gen) is LookupError
+            assert next(gen) is ValueError
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -87,6 +87,40 @@
         sys.settrace(None)
         # assert did not crash
 
+    def test_f_lineno_set_2(self):
+        counter = [0]
+        errors = []
+
+        def tracer(f, event, *args):
+            if event == 'line':
+                counter[0] += 1
+                if counter[0] == 2:
+                    try:
+                        f.f_lineno += 2
+                    except ValueError as e:
+                        errors.append(e)
+            return tracer
+
+        # obscure: call open beforehand, py3k's open invokes some app
+        # level code that confuses our tracing (likely due to the
+        # testing env, otherwise it's not a problem)
+        f = open(self.tempfile1, 'w')
+        def function():
+            try:
+                raise ValueError
+            except ValueError:
+                x = 42
+            return x
+
+        import sys
+        sys.settrace(tracer)
+        x = function()
+        sys.settrace(None)
+        assert x == 42
+        assert len(errors) == 1
+        assert str(errors[0]).startswith(
+            "can't jump into or out of an 'expect' or 'finally' block")
+
     def test_f_lineno_set_firstline(self):
         r"""
         seen = []
@@ -148,30 +182,6 @@
         assert f1bis is f1
         assert f0.f_back is f1
 
-    def test_f_exc_xxx(self):
-        import sys
-
-        class OuterException(Exception):
-            pass
-        class InnerException(Exception):
-            pass
-
-        def g(exc_info):
-            f = sys._getframe()
-            assert f.f_exc_type is None
-            assert f.f_exc_value is None
-            assert f.f_exc_traceback is None
-            try:
-                raise InnerException
-            except:
-                assert f.f_exc_type is exc_info[0]
-                assert f.f_exc_value is exc_info[1]
-                assert f.f_exc_traceback is exc_info[2]
-        try:
-            raise OuterException
-        except:
-            g(sys.exc_info())
-
     def test_virtualref_through_traceback(self):
         import sys
         def g():
diff --git a/pypy/interpreter/test/test_raise.py b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -79,6 +79,25 @@
             assert sys.exc_info()[0] is ValueError
         assert sys.exc_info() == (None, None, None)
 
+    def test_revert_exc_info_2_finally(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        try:
+            try:
+                raise ValueError
+            finally:
+                try:
+                    try:
+                        raise IndexError
+                    finally:
+                        assert sys.exc_info()[0] is IndexError
+                except IndexError:
+                    pass
+                assert sys.exc_info()[0] is ValueError
+        except ValueError:
+            pass
+        assert sys.exc_info() == (None, None, None)
+
     def test_reraise_1(self):
         raises(IndexError, """
             import sys
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -351,6 +351,7 @@
         assert list(result) == [2,3,4]
 
     def test_pickle_generator(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         import types
         mod = types.ModuleType('mod')
@@ -375,6 +376,7 @@
             del sys.modules['mod']
 
     def test_pickle_generator_blk(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         # same as above but with the generator inside a block
         import types
@@ -448,6 +450,7 @@
 
 
     def test_pickle_generator_crash(self):
+        skip("not supported any more for now")
         self.skip_on_cpython()
         import pickle
 
@@ -464,7 +467,7 @@
         assert 'finished' in repr(y)
         assert y.gi_code is None
 
-class AppTestGeneratorCloning:
+class XAppTestGeneratorCloning:
 
     def setup_class(cls):
         try:
@@ -528,7 +531,7 @@
             raises(StopIteration, next, g3)
             raises(StopIteration, next, g4)
 
-class AppTestFramePickling(object):
+class XAppTestFramePickling(object):
     pytestmark = py.test.mark.skipif("config.option.runappdirect")
     spaceconfig = {
         "usemodules": ["struct"]
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -464,7 +464,7 @@
 from pypy.interpreter.eval import Code
 from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS
 from pypy.interpreter.pyframe import PyFrame
-from pypy.interpreter.pyopcode import SuspendedUnroller, W_OperationError
+from pypy.interpreter.pyopcode import SuspendedUnroller
 from pypy.interpreter.module import Module
 from pypy.interpreter.function import (Function, Method, StaticMethod,
     ClassMethod, BuiltinFunction, descr_function_get)
@@ -607,8 +607,8 @@
 PyCode.typedef.acceptable_as_base_class = False
 
 PyFrame.typedef = TypeDef('frame',
-    __reduce__ = interp2app(PyFrame.descr__reduce__),
-    __setstate__ = interp2app(PyFrame.descr__setstate__),
+    #__reduce__ = interp2app(PyFrame.descr__reduce__),  --- logic not updated
+    #__setstate__ = interp2app(PyFrame.descr__setstate__),
     clear = interp2app(PyFrame.descr_clear),
     f_builtins = GetSetProperty(PyFrame.fget_f_builtins),
     f_lineno = GetSetProperty(PyFrame.fget_f_lineno, PyFrame.fset_f_lineno),
@@ -616,9 +616,6 @@
     f_lasti = GetSetProperty(PyFrame.fget_f_lasti),
     f_trace = GetSetProperty(PyFrame.fget_f_trace, PyFrame.fset_f_trace,
                              PyFrame.fdel_f_trace),
-    f_exc_type = GetSetProperty(PyFrame.fget_f_exc_type),
-    f_exc_value = GetSetProperty(PyFrame.fget_f_exc_value),
-    f_exc_traceback = GetSetProperty(PyFrame.fget_f_exc_traceback),
     f_restricted = GetSetProperty(PyFrame.fget_f_restricted),
     f_code = GetSetProperty(PyFrame.fget_code),
     f_locals = GetSetProperty(PyFrame.fget_getdictscope),
@@ -787,8 +784,8 @@
 
 GeneratorIterator.typedef = TypeDef("generator",
     __repr__   = interp2app(GeneratorIterator.descr__repr__),
-    __reduce__   = interp2app(GeneratorIterator.descr__reduce__),
-    __setstate__ = interp2app(GeneratorIterator.descr__setstate__),
+    #__reduce__   = interp2app(GeneratorIterator.descr__reduce__),
+    #__setstate__ = interp2app(GeneratorIterator.descr__setstate__),
     __next__   = interp2app(GeneratorIterator.descr_next,
                             descrmismatch='__next__'),
     send       = interp2app(GeneratorIterator.descr_send,
@@ -813,8 +810,8 @@
 
 Coroutine.typedef = TypeDef("coroutine",
     __repr__   = interp2app(Coroutine.descr__repr__),
-    __reduce__   = interp2app(Coroutine.descr__reduce__),
-    __setstate__ = interp2app(Coroutine.descr__setstate__),
+    #__reduce__   = interp2app(Coroutine.descr__reduce__),
+    #__setstate__ = interp2app(Coroutine.descr__setstate__),
     send       = interp2app(Coroutine.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(Coroutine.descr_throw,
@@ -878,8 +875,8 @@
 SuspendedUnroller.typedef = TypeDef("SuspendedUnroller")
 SuspendedUnroller.typedef.acceptable_as_base_class = False
 
-W_OperationError.typedef = TypeDef("OperationError",
-    __reduce__ = interp2app(W_OperationError.descr_reduce),
-    __setstate__ = interp2app(W_OperationError.descr_setstate),
-)
-W_OperationError.typedef.acceptable_as_base_class = False
+## W_OperationError.typedef = TypeDef("OperationError",
+##     __reduce__ = interp2app(W_OperationError.descr_reduce),
+##     __setstate__ = interp2app(W_OperationError.descr_setstate),
+## )
+## W_OperationError.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_pickle_support/__init__.py b/pypy/module/_pickle_support/__init__.py
--- a/pypy/module/_pickle_support/__init__.py
+++ b/pypy/module/_pickle_support/__init__.py
@@ -22,5 +22,4 @@
         'intrangeiter_new': 'maker.intrangeiter_new',
         'builtin_code': 'maker.builtin_code',
         'builtin_function' : 'maker.builtin_function',
-        'operationerror_new': 'maker.operationerror_new',
     }
diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -69,10 +69,6 @@
     new_iter = W_IntRangeIterator(space, current, remaining, step)
     return space.wrap(new_iter)
 
-def operationerror_new(space):
-    from pypy.interpreter.pyopcode import W_OperationError
-    return W_OperationError(None)
-
 @unwrap_spec(identifier=str)
 def builtin_code(space, identifier):
     from pypy.interpreter import gateway
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -26,7 +26,6 @@
                            'valuestackdepth',
                            'locals_cells_stack_w[*]',
                            'debugdata',
-                           'last_exception',
                            'lastblock',
                            'w_globals',
                            ]
@@ -98,7 +97,6 @@
             jit.hint(self, force_virtualizable=True)
             return w_result
         except Return:
-            self.last_exception = None
             return self.popvalue()
 
     def jump_absolute(self, jumpto, ec):
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -107,8 +107,7 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
-def exc_info_without_tb(space, frame):
-    operror = frame.last_exception
+def exc_info_without_tb(space, operror):
     return space.newtuple([operror.w_type, operror.get_w_value(space),
                            space.w_None])
 
@@ -152,10 +151,11 @@
                         if space.int_w(w_constant) <= 2:
                             need_all_three_args = False
     #
-    if need_all_three_args or frame.last_exception is None or frame.hide():
+    operror = space.getexecutioncontext().sys_exc_info()
+    if need_all_three_args or operror is None or frame.hide():
         return exc_info_with_tb(space)
     else:
-        return exc_info_without_tb(space, frame)
+        return exc_info_without_tb(space, operror)
 
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -210,26 +210,20 @@
                                space.newtuple([w_BuiltinAssertionError]),
                                w_dict)
 
-def _exc_info(space, err):
-    """Hack the fact that exc_info() isn't set until a app except
-    block catches it."""
-    err.normalize_exception(space)
-    frame = space.getexecutioncontext().gettopframe()
-    old = frame.last_exception
-    frame.last_exception = err
-    if not hasattr(space, '_w_ExceptionInfo'):
-        space._w_ExceptionInfo = space.appexec([], """():
-    class _ExceptionInfo(object):
-        def __init__(self):
-            import sys
-            self.type, self.value, self.traceback = sys.exc_info()
-
-    return _ExceptionInfo
-""")
-    try:
-        return space.call_function(space._w_ExceptionInfo)
-    finally:
-        frame.last_exception = old
+def _exc_info(space, operror):
+    """sys.exc_info() isn't set until a app except block catches it,
+    but we can directly copy the two lines of code from module/sys/vm.py."""
+    operror.normalize_exception(space)
+    return space.appexec([operror.w_type, operror.get_w_value(space),
+                          space.wrap(operror.get_traceback())], """(t, v, tb):
+        class _ExceptionInfo:
+            pass
+        e = _ExceptionInfo()
+        e.type = t
+        e.value = v
+        e.traceback = tb
+        return e
+    """)
 
 def pypyraises(space, w_ExpectedException, w_expr, __args__):
     """A built-in function providing the equivalent of py.test.raises()."""
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -172,7 +172,8 @@
         f.write('\n'.join(defs))
         f.write('def %s():\n' % target_name)
         f.write('\n'.join(source))
-        f.write("\n%s()\n" % target_name)
+        f.write("\ntry:\n    %s()\n" % target_name)
+        f.write('finally:\n    print("===aefwuiheawiu===")')
     helper_dir = os.path.join(pypydir, 'tool', 'cpyext')
     env = os.environ.copy()
     env['PYTHONPATH'] = helper_dir
@@ -186,6 +187,8 @@
                      (python_, usemodules))
     elif res > 0:
         raise AssertionError("Subprocess failed:\n" + stderr)
+    elif "===aefwuiheawiu===" not in stdout:
+        raise AssertionError("%r crashed:\n%s" % (python_, stderr))
 
 
 def extract_docstring_if_empty_function(fn):
@@ -209,6 +212,7 @@
 
     def execute_appex(self, space, target, *args):
         self.space = space
+        space.getexecutioncontext().set_sys_exc_info(None)
         try:
             target(*args)
         except OperationError as e:

From pypy.commits at gmail.com  Fri Nov 18 09:38:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 06:38:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Tweaks to this test file,
 to remove dependency on the test order
Message-ID: <582f1271.d5092e0a.5d9b0.88d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88467:1aaabb79a90f
Date: 2016-11-18 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/1aaabb79a90f/

Log:	Tweaks to this test file, to remove dependency on the test order
	(shown by test_blocking which used to fail if we run all tests)

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -558,11 +558,10 @@
 
     def test_utime(self):
         os = self.posix
-        from os.path import join
         # XXX utimes & float support
-        path = join(self.pdir, "test_utime.txt")
-        fh = open(path, "w")
-        fh.write("x")
+        path = self.path2 + "test_utime.txt"
+        fh = open(path, "wb")
+        fh.write(b"x")
         fh.close()
         from time import time, sleep
         t0 = time()
@@ -907,53 +906,51 @@
 
     if hasattr(os, 'chown'):
         def test_chown(self):
+            my_path = self.path2 + 'test_chown'
             os = self.posix
-            os.unlink(self.path)
-            raises(OSError, os.chown, self.path, os.getuid(), os.getgid())
-            f = open(self.path, "w")
-            f.write("this is a test")
-            f.close()
-            os.chown(self.path, os.getuid(), os.getgid())
+            raises(OSError, os.chown, my_path, os.getuid(), os.getgid())
+            open(my_path, 'w').close()
+            os.chown(my_path, os.getuid(), os.getgid())
 
     if hasattr(os, 'lchown'):
         def test_lchown(self):
+            my_path = self.path2 + 'test_lchown'
             os = self.posix
-            os.unlink(self.path)
-            raises(OSError, os.lchown, self.path, os.getuid(), os.getgid())
-            os.symlink('foobar', self.path)
-            os.lchown(self.path, os.getuid(), os.getgid())
+            raises(OSError, os.lchown, my_path, os.getuid(), os.getgid())
+            os.symlink('foobar', my_path)
+            os.lchown(my_path, os.getuid(), os.getgid())
 
     if hasattr(os, 'fchown'):
         def test_fchown(self):
+            my_path = self.path2 + 'test_fchown'
             os = self.posix
-            f = open(self.path, "w")
+            f = open(my_path, "w")
             os.fchown(f.fileno(), os.getuid(), os.getgid())
             f.close()
 
     if hasattr(os, 'chmod'):
         def test_chmod(self):
             import sys
+            my_path = self.path2 + 'test_chmod'
             os = self.posix
-            os.unlink(self.path)
-            raises(OSError, os.chmod, self.path, 0o600)
-            f = open(self.path, "w")
-            f.write("this is a test")
-            f.close()
+            raises(OSError, os.chmod, my_path, 0o600)
+            open(my_path, "w").close()
             if sys.platform == 'win32':
-                os.chmod(self.path, 0o400)
-                assert (os.stat(self.path).st_mode & 0o600) == 0o400
+                os.chmod(my_path, 0o400)
+                assert (os.stat(my_path).st_mode & 0o600) == 0o400
             else:
-                os.chmod(self.path, 0o200)
-                assert (os.stat(self.path).st_mode & 0o777) == 0o200
+                os.chmod(my_path, 0o200)
+                assert (os.stat(my_path).st_mode & 0o777) == 0o200
 
     if hasattr(os, 'fchmod'):
         def test_fchmod(self):
+            my_path = self.path2 + 'test_fchmod'
             os = self.posix
-            f = open(self.path, "w")
+            f = open(my_path, "w")
             os.fchmod(f.fileno(), 0o200)
             assert (os.fstat(f.fileno()).st_mode & 0o777) == 0o200
             f.close()
-            assert (os.stat(self.path).st_mode & 0o777) == 0o200
+            assert (os.stat(my_path).st_mode & 0o777) == 0o200
 
     if hasattr(os, 'mkfifo'):
         def test_mkfifo(self):

From pypy.commits at gmail.com  Fri Nov 18 10:12:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 07:12:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Avoid relying on pytest's rarely-used
 functions (this is unrelated to
Message-ID: <582f1a44.0a4b2e0a.b47fb.8b5d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88468:b3d7eec2a80e
Date: 2016-11-18 16:11 +0100
http://bitbucket.org/pypy/pypy/changeset/b3d7eec2a80e/

Log:	Avoid relying on pytest's rarely-used functions (this is unrelated
	to the update to pytest-2.9.2): here, testdir.inline_run(), which
	fails in the py3.5 branch for no reason I want to understand.

diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -1,61 +1,78 @@
 
 import py
 import sys
+import subprocess
 
 innertest = py.path.local(__file__).dirpath('conftest1_innertest.py')
-pytest_plugins = "pytester"
+#pytest_plugins = "pytester"
+
+
+def subproc_run(*args):
+    # similar to testdir.inline_run(), but running a subprocess to avoid
+    # confusion.  Parses the standard output of test_all.py, so somehow
+    # dependent on how it looks like.
+    test_all = py.path.local(__file__).dirpath('..', '..', '..', 'test_all.py')
+    args = [sys.executable, str(test_all), "-v"] + map(str, args)
+    print '=========>', args
+    passed, failed = [], []
+    popen = subprocess.Popen(args, stdout=subprocess.PIPE)
+    output, _ = popen.communicate()
+
+    for line in output.splitlines(False):
+        if line.startswith('conftest1_innertest.py'):
+            line = line[len('conftest1_innertest.py'):]
+            testname, result = line.lstrip(':').strip().split()
+            if result == 'PASSED':
+                passed.append(testname)
+            elif result == 'FAILED':
+                failed.append(testname)
+            else:
+                assert False, "unexpected output line: %r" % (line,)
+    return passed, failed
+
 
 class TestPyPyTests:
-    def test_selection_by_keyword_interp(self, testdir):
-        sorter = testdir.inline_run("-m", "interplevel", innertest, )
-        passed, skipped, failed = sorter.listoutcomes()
+    def test_selection_by_keyword_interp(self):
+        passed, failed = subproc_run("-m", "interplevel", innertest)
         assert len(passed) == 2, len(passed)
-        assert not skipped and not failed
-        assert "test_something" in passed[0].nodeid
-        assert "test_method" in passed[1].nodeid
+        assert not failed
+        assert "test_something" in passed[0]
+        assert "test_method" in passed[1]
 
-    def test_selection_by_keyword_app(self, testdir):
-        sorter = testdir.inline_run("-m", "applevel -docstring", innertest)
-        passed, skipped, failed = sorter.listoutcomes()
+    def test_selection_by_keyword_app(self):
+        passed, failed = subproc_run("-m", "applevel -docstring", innertest)
         assert len(passed) == 4
-        assert not skipped
         assert len(failed) == 2
-        assert "app_test_something" in passed[0].nodeid
-        assert "test_method_app" in passed[1].nodeid
-        
-    def test_docstring_in_methods(self, testdir): 
-        sorter = testdir.inline_run("-k", "AppTestSomething and test_code_in_docstring",
+        assert "app_test_something" in passed[0]
+        assert "test_method_app" in passed[1]
+
+    def test_docstring_in_methods(self):
+        passed, failed = subproc_run("-k", "AppTestSomething and test_code_in_docstring",
                                     innertest)
-        passed, skipped, failed = sorter.listoutcomes()
         assert len(passed) == 1
         assert len(failed) == 1
-        assert skipped == []
-        assert "test_code_in_docstring_ignored" in passed[0].nodeid
-        assert "test_code_in_docstring_failing" in failed[0].nodeid
+        assert "test_code_in_docstring_ignored" in passed[0]
+        assert "test_code_in_docstring_failing" in failed[0]
 
-    def test_docstring_in_functions(self, testdir): 
-        sorter = testdir.inline_run("-k", "app_test_code_in_docstring", innertest)
-        passed, skipped, failed = sorter.listoutcomes()
+    def test_docstring_in_functions(self):
+        passed, failed = subproc_run("-k", "app_test_code_in_docstring", innertest)
         assert passed == []
         assert len(failed) == 1
-        assert skipped == []
-        assert "app_test_code_in_docstring_failing" in failed[0].nodeid
+        assert "app_test_code_in_docstring_failing" in failed[0]
 
-    def test_docstring_runappdirect(self, testdir):
-        sorter = testdir.inline_run(innertest,
+    def test_docstring_runappdirect(self):
+        passed, failed = subproc_run(innertest,
                                     '-k', 'test_code_in_docstring',
                                     '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
         assert len(passed) == 1
         assert len(failed) == 2
-        assert "test_code_in_docstring_ignored" in passed[0].nodeid
-        assert "app_test_code_in_docstring_failing" in failed[0].nodeid
-        assert "test_code_in_docstring_failing" in failed[1].nodeid
+        assert "test_code_in_docstring_ignored" in passed[0]
+        assert "app_test_code_in_docstring_failing" in failed[0]
+        assert "test_code_in_docstring_failing" in failed[1]
 
-    def test_raises_inside_closure(self, testdir):
-        sorter = testdir.inline_run(innertest, '-k', 'app_test_raise_in_a_closure',
+    def test_raises_inside_closure(self):
+        passed, failed = subproc_run(innertest, '-k', 'app_test_raise_in_a_closure',
                                     '--runappdirect')
-        passed, skipped, failed = sorter.listoutcomes()
         assert len(passed) == 1
         print passed
-        assert "app_test_raise_in_a_closure" in passed[0].nodeid
+        assert "app_test_raise_in_a_closure" in passed[0]

From pypy.commits at gmail.com  Fri Nov 18 10:33:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 07:33:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: In this test,
 interpret() seems not to be usable on non-assert
Message-ID: <582f1f64.d4c4190a.c213.8dea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88469:cbe653eeb8fb
Date: 2016-11-18 16:33 +0100
http://bitbucket.org/pypy/pypy/changeset/cbe653eeb8fb/

Log:	In this test, interpret() seems not to be usable on non-assert
	statements. Do the 'f = lambda x: x+1' manually instead.

diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -21,7 +21,13 @@
     co = PyCode._from_code(space, somefunc.func_code)
     pyframe = space.FrameClass(space, co, space.newdict(), None)
     runner = AppFrame(space, pyframe)
-    interpret("f = lambda x: x+1", runner, should_fail=False)
+    # XXX the following, in two calls to interpret(), no longer seems to
+    # work---the name 'f' is not defined in the second call.  We must
+    # build the 'f' manually with a space.appexec().
+    #interpret("f = lambda x: x+1", runner, should_fail=False)
+    space.appexec([runner.get_w_globals()], """(dict):
+        dict['f'] = lambda x: x+1
+    """)
     msg = interpret("assert isinstance(f(2), float)", runner)
     assert msg.startswith("assert isinstance(3, float)\n"
                           " +  where 3 = ")

From pypy.commits at gmail.com  Fri Nov 18 10:44:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 07:44:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: bytearray.__alloc__()
Message-ID: <582f21f5.8786190a.3ee1b.8e7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88470:91461751edba
Date: 2016-11-18 16:44 +0100
http://bitbucket.org/pypy/pypy/changeset/91461751edba/

Log:	bytearray.__alloc__()

diff --git a/lib-python/3/test/test_bytes.py b/lib-python/3/test/test_bytes.py
--- a/lib-python/3/test/test_bytes.py
+++ b/lib-python/3/test/test_bytes.py
@@ -1095,7 +1095,6 @@
         self.assertEqual(b, b1)
         self.assertTrue(b is b1)
 
-    @test.support.impl_detail("undocumented bytes.__alloc__()")
     def test_alloc(self):
         b = bytearray()
         alloc = b.__alloc__()
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -452,6 +452,9 @@
         assert isinstance(self, W_BytearrayObject)
         return self._getitem_result(space, index)
 
+    def descr_alloc(self, space):
+        return space.wrap(self._len() + 1)
+
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
@@ -552,7 +555,9 @@
     def __alloc__():
         """B.__alloc__() -> int
 
-        Return the number of bytes actually allocated.
+        CPython compatibility: return len(B) + 1.
+        (The allocated size might be bigger, but getting it is
+        involved and may create pointless compatibility troubles.)
         """
 
     def __contains__():
@@ -967,6 +972,7 @@
 
     def hex():
         """B.hex() -> unicode
+
         Return a string object containing two hexadecimal digits
         for each byte in the instance B.
         """
@@ -1124,6 +1130,8 @@
                          doc=BytearrayDocstrings.copy.__doc__),
     hex = interp2app(W_BytearrayObject.descr_hex,
                            doc=BytearrayDocstrings.hex.__doc__),
+    __alloc__ = interp2app(W_BytearrayObject.descr_alloc,
+                           doc=BytearrayDocstrings.__alloc__.__doc__),
 )
 W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -548,3 +548,7 @@
 
     def test_format_bytes(self):
         assert bytearray(b'<%s>') % b'abc' == b'<abc>'
+
+    def test___alloc__(self):
+        # pypy: always returns len()+1; cpython: may be bigger
+        assert bytearray(b'123456').__alloc__() >= 7

From pypy.commits at gmail.com  Fri Nov 18 11:05:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 08:05:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the docstrings of bytes
Message-ID: <582f26b4.8f19190a.e6ebe.9195@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88471:b8860edf230d
Date: 2016-11-18 16:55 +0100
http://bitbucket.org/pypy/pypy/changeset/b8860edf230d/

Log:	Fix the docstrings of bytes

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -102,32 +102,32 @@
         """x.__str__() <==> str(x)"""
 
     def descr_capitalize(self, space):
-        """S.capitalize() -> string
+        """B.capitalize() -> copy of B
 
-        Return a capitalized version of S, i.e. make the first character
+        Return a capitalized version of B, i.e. make the first character
         have upper case and the rest lower case.
         """
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_center(self, space, width, w_fillchar):
-        """S.center(width[, fillchar]) -> string
+        """B.center(width[, fillchar]) -> copy of B
 
-        Return S centered in a string of length width. Padding is
+        Return B centered in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
-        """S.count(sub[, start[, end]]) -> int
+        """B.count(sub[, start[, end]]) -> int
 
         Return the number of non-overlapping occurrences of substring sub in
-        string S[start:end].  Optional arguments start and end are interpreted
+        string B[start:end].  Optional arguments start and end are interpreted
         as in slice notation.
         """
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
-        """S.decode(encoding=None, errors='strict') -> object
+        """B.decode(encoding=None, errors='strict') -> object
 
-        Decode S using the codec registered for encoding. encoding defaults
+        Decode B using the codec registered for encoding. encoding defaults
         to the default encoding. errors may be given to set a different error
         handling scheme. Default is 'strict' meaning that encoding errors raise
         a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'
@@ -136,172 +136,171 @@
         """
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
-        """S.endswith(suffix[, start[, end]]) -> bool
+        """B.endswith(suffix[, start[, end]]) -> bool
 
-        Return True if S ends with the specified suffix, False otherwise.
-        With optional start, test S beginning at that position.
-        With optional end, stop comparing S at that position.
-        suffix can also be a tuple of strings to try.
+        Return True if B ends with the specified suffix, False otherwise.
+        With optional start, test B beginning at that position.
+        With optional end, stop comparing B at that position.
+        suffix can also be a tuple of bytes to try.
         """
 
     @unwrap_spec(tabsize=int)
     def descr_expandtabs(self, space, tabsize=8):
-        """S.expandtabs([tabsize]) -> string
+        """B.expandtabs([tabsize]) -> copy of B
 
-        Return a copy of S where all tab characters are expanded using spaces.
+        Return a copy of B where all tab characters are expanded using spaces.
         If tabsize is not given, a tab size of 8 characters is assumed.
         """
 
     def descr_find(self, space, w_sub, w_start=None, w_end=None):
-        """S.find(sub[, start[, end]]) -> int
+        """B.find(sub[, start[, end]]) -> int
 
-        Return the lowest index in S where substring sub is found,
-        such that sub is contained within S[start:end].  Optional
+        Return the lowest index in B where substring sub is found,
+        such that sub is contained within B[start:end].  Optional
         arguments start and end are interpreted as in slice notation.
 
         Return -1 on failure.
         """
 
     def descr_index(self, space, w_sub, w_start=None, w_end=None):
-        """S.index(sub[, start[, end]]) -> int
+        """B.index(sub[, start[, end]]) -> int
 
-        Like S.find() but raise ValueError when the substring is not found.
+        Like B.find() but raise ValueError when the substring is not found.
         """
 
     def descr_isalnum(self, space):
-        """S.isalnum() -> bool
+        """B.isalnum() -> bool
 
-        Return True if all characters in S are alphanumeric
-        and there is at least one character in S, False otherwise.
+        Return True if all characters in B are alphanumeric
+        and there is at least one character in B, False otherwise.
         """
 
     def descr_isalpha(self, space):
-        """S.isalpha() -> bool
+        """B.isalpha() -> bool
 
-        Return True if all characters in S are alphabetic
-        and there is at least one character in S, False otherwise.
+        Return True if all characters in B are alphabetic
+        and there is at least one character in B, False otherwise.
         """
 
     def descr_isdigit(self, space):
-        """S.isdigit() -> bool
+        """B.isdigit() -> bool
 
-        Return True if all characters in S are digits
-        and there is at least one character in S, False otherwise.
+        Return True if all characters in B are digits
+        and there is at least one character in B, False otherwise.
         """
 
     def descr_islower(self, space):
-        """S.islower() -> bool
+        """B.islower() -> bool
 
-        Return True if all cased characters in S are lowercase and there is
-        at least one cased character in S, False otherwise.
+        Return True if all cased characters in B are lowercase and there is
+        at least one cased character in B, False otherwise.
         """
 
     def descr_isspace(self, space):
-        """S.isspace() -> bool
+        """B.isspace() -> bool
 
-        Return True if all characters in S are whitespace
-        and there is at least one character in S, False otherwise.
+        Return True if all characters in B are whitespace
+        and there is at least one character in B, False otherwise.
         """
 
     def descr_istitle(self, space):
-        """S.istitle() -> bool
+        """B.istitle() -> bool
 
-        Return True if S is a titlecased string and there is at least one
-        character in S, i.e. uppercase characters may only follow uncased
+        Return True if B is a titlecased string and there is at least one
+        character in B, i.e. uppercase characters may only follow uncased
         characters and lowercase characters only cased ones. Return False
         otherwise.
         """
 
     def descr_isupper(self, space):
-        """S.isupper() -> bool
+        """B.isupper() -> bool
 
-        Return True if all cased characters in S are uppercase and there is
-        at least one cased character in S, False otherwise.
+        Return True if all cased characters in B are uppercase and there is
+        at least one cased character in B, False otherwise.
         """
 
     def descr_join(self, space, w_list):
-        """S.join(iterable) -> string
+        """B.join(iterable) -> bytes
 
-        Return a string which is the concatenation of the strings in the
-        iterable.  The separator between elements is S.
+        Return a bytes object which is the concatenation of the bytes in the
+        iterable.  The separator between elements is B.
         """
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_ljust(self, space, width, w_fillchar):
-        """S.ljust(width[, fillchar]) -> string
+        """B.ljust(width[, fillchar]) -> copy of B
 
-        Return S left-justified in a string of length width. Padding is
+        Return B left-justified in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
 
     def descr_lower(self, space):
-        """S.lower() -> string
+        """B.lower() -> copy of B
 
-        Return a copy of the string S converted to lowercase.
+        Return a copy of the string B converted to lowercase.
         """
 
     def descr_lstrip(self, space, w_chars=None):
-        """S.lstrip([chars]) -> string or unicode
+        """B.lstrip([chars]) -> copy of B
 
-        Return a copy of the string S with leading whitespace removed.
+        Return a copy of the string B with leading whitespace removed.
         If chars is given and not None, remove characters in chars instead.
-        If chars is unicode, S will be converted to unicode before stripping
         """
 
     def descr_partition(self, space, w_sub):
-        """S.partition(sep) -> (head, sep, tail)
+        """B.partition(sep) -> (head, sep, tail)
 
-        Search for the separator sep in S, and return the part before it,
+        Search for the separator sep in B, and return the part before it,
         the separator itself, and the part after it.  If the separator is not
-        found, return S and two empty strings.
+        found, return B and two empty bytes objects.
         """
 
     @unwrap_spec(count=int)
     def descr_replace(self, space, w_old, w_new, count=-1):
-        """S.replace(old, new[, count]) -> string
+        """B.replace(old, new[, count]) -> copy of B
 
-        Return a copy of string S with all occurrences of substring
+        Return a copy of string B with all occurrences of substring
         old replaced by new.  If the optional argument count is
         given, only the first count occurrences are replaced.
         """
 
     def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
-        """S.rfind(sub[, start[, end]]) -> int
+        """B.rfind(sub[, start[, end]]) -> int
 
-        Return the highest index in S where substring sub is found,
-        such that sub is contained within S[start:end].  Optional
+        Return the highest index in B where substring sub is found,
+        such that sub is contained within B[start:end].  Optional
         arguments start and end are interpreted as in slice notation.
 
         Return -1 on failure.
         """
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
-        """S.rindex(sub[, start[, end]]) -> int
+        """B.rindex(sub[, start[, end]]) -> int
 
-        Like S.rfind() but raise ValueError when the substring is not found.
+        Like B.rfind() but raise ValueError when the substring is not found.
         """
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_rjust(self, space, width, w_fillchar):
-        """S.rjust(width[, fillchar]) -> string
+        """B.rjust(width[, fillchar]) -> copy of B
 
-        Return S right-justified in a string of length width. Padding is
+        Return B right-justified in a string of length width. Padding is
         done using the specified fill character (default is a space).
         """
 
     def descr_rpartition(self, space, w_sub):
-        """S.rpartition(sep) -> (head, sep, tail)
+        """B.rpartition(sep) -> (head, sep, tail)
 
-        Search for the separator sep in S, starting at the end of S, and return
+        Search for the separator sep in B, starting at the end of B, and return
         the part before it, the separator itself, and the part after it.  If
-        the separator is not found, return two empty strings and S.
+        the separator is not found, return two empty bytes objects and B.
         """
 
     @unwrap_spec(maxsplit=int)
     def descr_rsplit(self, space, w_sep=None, maxsplit=-1):
-        """S.rsplit(sep=None, maxsplit=-1) -> list of strings
+        """B.rsplit(sep=None, maxsplit=-1) -> list of bytes objects
 
-        Return a list of the words in the string S, using sep as the
+        Return a list of the words in the string B, using sep as the
         delimiter string, starting at the end of the string and working
         to the front.  If maxsplit is given, at most maxsplit splits are
         done. If sep is not specified or is None, any whitespace string
@@ -309,18 +308,17 @@
         """
 
     def descr_rstrip(self, space, w_chars=None):
-        """S.rstrip([chars]) -> string or unicode
+        """B.rstrip([chars]) -> copy of B
 
-        Return a copy of the string S with trailing whitespace removed.
+        Return a copy of the string B with trailing whitespace removed.
         If chars is given and not None, remove characters in chars instead.
-        If chars is unicode, S will be converted to unicode before stripping
         """
 
     @unwrap_spec(maxsplit=int)
     def descr_split(self, space, w_sep=None, maxsplit=-1):
-        """S.split(sep=None, maxsplit=-1) -> list of strings
+        """B.split(sep=None, maxsplit=-1) -> list of bytes objects
 
-        Return a list of the words in the string S, using sep as the
+        Return a list of the words in the string B, using sep as the
         delimiter string.  If maxsplit is given, at most maxsplit
         splits are done. If sep is not specified or is None, any
         whitespace string is a separator and empty strings are removed
@@ -329,50 +327,49 @@
 
     @unwrap_spec(keepends=bool)
     def descr_splitlines(self, space, keepends=False):
-        """S.splitlines(keepends=False) -> list of strings
+        """B.splitlines(keepends=False) -> list of bytes objects
 
-        Return a list of the lines in S, breaking at line boundaries.
+        Return a list of the lines in B, breaking at line boundaries.
         Line breaks are not included in the resulting list unless keepends
         is given and true.
         """
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
-        """S.startswith(prefix[, start[, end]]) -> bool
+        """B.startswith(prefix[, start[, end]]) -> bool
 
-        Return True if S starts with the specified prefix, False otherwise.
-        With optional start, test S beginning at that position.
-        With optional end, stop comparing S at that position.
-        prefix can also be a tuple of strings to try.
+        Return True if B starts with the specified prefix, False otherwise.
+        With optional start, test B beginning at that position.
+        With optional end, stop comparing B at that position.
+        prefix can also be a tuple of bytes to try.
         """
 
     def descr_strip(self, space, w_chars=None):
-        """S.strip([chars]) -> string or unicode
+        """B.strip([chars]) -> copy of B
 
-        Return a copy of the string S with leading and trailing
+        Return a copy of the string B with leading and trailing
         whitespace removed.
         If chars is given and not None, remove characters in chars instead.
-        If chars is unicode, S will be converted to unicode before stripping
         """
 
     def descr_swapcase(self, space):
-        """S.swapcase() -> string
+        """B.swapcase() -> copy of B
 
-        Return a copy of the string S with uppercase characters
+        Return a copy of the string B with uppercase characters
         converted to lowercase and vice versa.
         """
 
     def descr_title(self, space):
-        """S.title() -> string
+        """B.title() -> copy of B
 
-        Return a titlecased version of S, i.e. words start with uppercase
+        Return a titlecased version of B, i.e. words start with uppercase
         characters, all remaining cased characters have lowercase.
         """
 
     @unwrap_spec(w_deletechars=WrappedDefault(''))
     def descr_translate(self, space, w_table, w_deletechars):
-        """S.translate(table[, deletechars]) -> string
+        """B.translate(table[, deletechars]) -> copy of B
 
-        Return a copy of the string S, where all characters occurring
+        Return a copy of the string B, where all characters occurring
         in the optional argument deletechars are removed, and the
         remaining characters have been mapped through the given
         translation table, which must be a string of length 256 or None.
@@ -381,21 +378,21 @@
         """
 
     def descr_upper(self, space):
-        """S.upper() -> string
+        """B.upper() -> copy of B
 
-        Return a copy of the string S converted to uppercase.
+        Return a copy of the string B converted to uppercase.
         """
 
     @unwrap_spec(width=int)
     def descr_zfill(self, space, width):
-        """S.zfill(width) -> string
+        """B.zfill(width) -> copy of B
 
-        Pad a numeric string S with zeros on the left, to fill a field
-        of the specified width. The string S is never truncated.
+        Pad a numeric string B with zeros on the left, to fill a field
+        of the specified width. The string B is never truncated.
         """
 
     def descr_mod(self, space, w_values):
-        """S % values -> string
+        """B % values -> bytes
 
         Format bytes objects
         """

From pypy.commits at gmail.com  Fri Nov 18 11:05:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 08:05:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix error messages of some
 bytes/bytearray operations
Message-ID: <582f26b6.2611190a.180ee.917c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88472:f930c9e46e70
Date: 2016-11-18 17:04 +0100
http://bitbucket.org/pypy/pypy/changeset/f930c9e46e70/

Log:	Fix error messages of some bytes/bytearray operations

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -27,6 +27,8 @@
 
 class W_BytearrayObject(W_Root):
     import_from_mixin(StringMethods)
+    _KIND1 = "bytearray"
+    _KIND2 = "bytearray"
 
     def __init__(self, data):
         check_list_of_chars(data)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -400,6 +400,8 @@
 class W_BytesObject(W_AbstractBytesObject):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
+    _KIND1 = "byte"
+    _KIND2 = "bytes"
 
     def __init__(self, str):
         assert str is not None
@@ -424,7 +426,7 @@
 
     def writebuf_w(self, space):
         raise oefmt(space.w_TypeError,
-                    "Cannot use string as modifiable buffer")
+                    "Cannot use bytes as modifiable buffer")
 
     def descr_getbuffer(self, space, w_flags):
         #from pypy.objspace.std.bufferobject import W_Buffer
@@ -437,7 +439,7 @@
     def ord(self, space):
         if len(self._value) != 1:
             raise oefmt(space.w_TypeError,
-                        "ord() expected a character, but string of length %d "
+                        "ord() expected a character, but bytes of length %d "
                         "found", len(self._value))
         return space.wrap(ord(self._value[0]))
 
@@ -756,7 +758,7 @@
 
     if space.isinstance_w(w_source, space.w_unicode):
         raise oefmt(space.w_TypeError,
-                    "cannot convert unicode object to bytes")
+                    "cannot convert a (unicode) str object to bytes")
 
     # sequence of bytes
     w_iter = space.iter(w_source)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -116,6 +116,9 @@
 
     descr_rmul = descr_mul
 
+    _KIND1 = "string"
+    _KIND2 = "string"
+
     def descr_getitem(self, space, w_index):
         if isinstance(w_index, W_SliceObject):
             selfvalue = self._val(space)
@@ -130,7 +133,7 @@
                 ret = _descr_getslice_slowpath(selfvalue, start, step, sl)
                 return self._new_from_list(ret)
 
-        index = space.getindex_w(w_index, space.w_IndexError, "string")
+        index = space.getindex_w(w_index, space.w_IndexError, self._KIND1)
         return self._getitem_result(space, index)
 
     def _getitem_result(self, space, index):
@@ -140,7 +143,7 @@
         try:
             character = selfvalue[index]
         except IndexError:
-            raise oefmt(space.w_IndexError, "string index out of range")
+            raise oefmt(space.w_IndexError, self._KIND1 + " index out of range")
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(self, W_BytesObject):
             return space.wrap(ord(character))
@@ -274,7 +277,7 @@
 
         if res < 0:
             raise oefmt(space.w_ValueError,
-                        "substring not found in string.index")
+                        "substring not found in " + self._KIND2 + ".index")
         return space.wrap(res)
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
@@ -288,7 +291,7 @@
 
         if res < 0:
             raise oefmt(space.w_ValueError,
-                        "substring not found in string.rindex")
+                        "substring not found in " + self._KIND2 + ".rindex")
         return space.wrap(res)
 
     @specialize.arg(2)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -552,3 +552,8 @@
     def test___alloc__(self):
         # pypy: always returns len()+1; cpython: may be bigger
         assert bytearray(b'123456').__alloc__() >= 7
+
+    def test_getitem_error_message(self):
+        e = raises(TypeError, bytearray(b'abc').__getitem__, b'd')
+        assert str(e.value).startswith(
+            'bytearray indices must be integers or slices')
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -931,3 +931,8 @@
         raises(TypeError, 'b"%b" % "hello world"')
         assert b'%b %b' % (b'a', bytearray(b'f f e')) == b'a f f e'
         """
+
+    def test_getitem_error_message(self):
+        e = raises(TypeError, b'abc'.__getitem__, b'd')
+        assert str(e.value).startswith(
+            'byte indices must be integers or slices')

From pypy.commits at gmail.com  Fri Nov 18 11:20:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 08:20:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Duplicate bytes's 'test_compatibility'
 for bytearray. Fix the
Message-ID: <582f2a60.46c6190a.209ce.915f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88473:ba9187343873
Date: 2016-11-18 17:20 +0100
http://bitbucket.org/pypy/pypy/changeset/ba9187343873/

Log:	Duplicate bytes's 'test_compatibility' for bytearray. Fix the case
	of bytearray.join().

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -150,10 +150,6 @@
     def _join_return_one(self, space, w_obj):
         return False
 
-    def _join_check_item(self, space, w_obj):
-        return not (space.isinstance_w(w_obj, space.w_bytes) or
-                    space.isinstance_w(w_obj, space.w_bytearray))
-
     def ord(self, space):
         if len(self.data) != 1:
             raise oefmt(space.w_TypeError,
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -649,15 +649,6 @@
     def _join_return_one(self, space, w_obj):
         return space.is_w(space.type(w_obj), space.w_str)
 
-    def _join_check_item(self, space, w_obj):
-        try:
-            self._op_val(space, w_obj)
-        except OperationError as e:
-            if not e.match(space, space.w_TypeError):
-                raise
-            return True
-        return False
-
     def descr_lower(self, space):
         return W_BytesObject(self._value.lower())
 
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -407,13 +407,17 @@
         unwrapped = newlist_hint(size)
         for i in range(size):
             w_s = list_w[i]
-            if self._join_check_item(space, w_s):
+            try:
+                next_string = self._op_val(space, w_s)
+            except OperationError as e:
+                if not e.match(space, space.w_TypeError):
+                    raise
                 raise oefmt(space.w_TypeError,
                             "sequence item %d: expected %s, %T found",
                             i, self._generic_name(), w_s)
             # XXX Maybe the extra copy here is okay? It was basically going to
             #     happen anyway, what with being placed into the builder
-            unwrapped.append(self._op_val(space, w_s))
+            unwrapped.append(next_string)
             prealloc_size += len(unwrapped[i])
 
         sb = self._builder(prealloc_size)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -557,3 +557,26 @@
         e = raises(TypeError, bytearray(b'abc').__getitem__, b'd')
         assert str(e.value).startswith(
             'bytearray indices must be integers or slices')
+
+    def test_compatibility(self):
+        # see comments in test_bytesobject.test_compatibility
+        b = bytearray(b'hello world')
+        b2 = b'ello'
+        #not testing result, just lack of TypeError
+        for bb in (b2, bytearray(b2), memoryview(b2)):
+            assert b.split(bb)
+            assert b.rsplit(bb)
+            assert b.split(bb[:1])
+            assert b.rsplit(bb[:1])
+            assert b.join((bb, bb))
+            assert bb in b
+            assert b.find(bb)
+            assert b.rfind(bb)
+            assert b.strip(bb)
+            assert b.rstrip(bb)
+            assert b.lstrip(bb)
+            assert not b.startswith(bb)
+            assert not b.startswith((bb, bb))
+            assert not b.endswith(bb)
+            assert not b.endswith((bb, bb))
+            assert bytearray.maketrans(bb, bb)
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -887,8 +887,7 @@
             assert b.rsplit(bb)
             assert b.split(bb[:1])
             assert b.rsplit(bb[:1])
-            assert b.join((bb, bb)) # cpython accepts bytes and
-                                    # bytearray only, not buffer
+            assert b.join((bb, bb))  # accepts memoryview() since CPython 3.4/5
             assert bb in b
             assert b.find(bb)
             assert b.rfind(bb)
@@ -900,7 +899,6 @@
             assert not b.endswith(bb)
             assert not b.endswith((bb, bb))
             assert bytes.maketrans(bb, bb)
-            assert bytearray.maketrans(bb, bb)
 
     def test_constructor_dont_convert_int(self):
         class A(object):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -457,9 +457,6 @@
     def _join_return_one(self, space, w_obj):
         return space.is_w(space.type(w_obj), space.w_unicode)
 
-    def _join_check_item(self, space, w_obj):
-        return not space.isinstance_w(w_obj, space.w_unicode)
-
     def descr_casefold(self, space):
         value = self._val(space)
         builder = self._builder(len(value))

From pypy.commits at gmail.com  Fri Nov 18 11:39:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 08:39:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Speed up 'bytes(unicode,
 encoding)' and systematically check that
Message-ID: <582f2ebe.a118190a.64378.916b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88474:e6763b1bfc48
Date: 2016-11-18 17:38 +0100
http://bitbucket.org/pypy/pypy/changeset/e6763b1bfc48/

Log:	Speed up 'bytes(unicode, encoding)' and systematically check that no
	encoding is given if the argument is not a unicode

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -530,7 +530,8 @@
     def descr_new(space, w_stringtype, w_source=None, encoding=None,
                   errors=None):
         if (w_source and space.is_w(space.type(w_source), space.w_bytes) and
-            space.is_w(w_stringtype, space.w_bytes)):
+            space.is_w(w_stringtype, space.w_bytes) and encoding is None
+            and errors is None):
             return w_source
         value = newbytesdata_w(space, w_source, encoding, errors)
         w_obj = space.allocate_instance(W_BytesObject, w_stringtype)
@@ -687,13 +688,28 @@
     return chr(value)
 
 def newbytesdata_w(space, w_source, encoding, errors):
+    # Unicode with encoding
+    if w_source is not None and space.isinstance_w(w_source, space.w_unicode):
+        if encoding is None:
+            raise oefmt(space.w_TypeError,
+                        "string argument without an encoding")
+        from pypy.objspace.std.unicodeobject import encode_object
+        w_source = encode_object(space, w_source, encoding, errors)
+        # and continue with the encoded string
+    elif encoding is not None or errors is not None:
+        if w_source is None:
+            raise oefmt(space.w_TypeError,
+                        "encoding or errors without string argument")
+        raise oefmt(space.w_TypeError,
+               "encoding or errors without string argument (got '%T' instead)",
+               w_source)
     # None value
     if w_source is None:
-        if encoding is not None or errors is not None:
-            raise oefmt(space.w_TypeError,
-                        "encoding or errors without string argument")
         return b""
-    # Some object with __bytes__ special method
+    # Fast-path for bytes
+    if space.isinstance_w(w_source, space.w_str):
+        return space.bytes_w(w_source)
+    # Some other object with a __bytes__ special method
     w_bytes_method = space.lookup(w_source, "__bytes__")
     if w_bytes_method is not None:
         w_bytes = space.get_and_call_function(w_bytes_method, w_source)
@@ -711,19 +727,7 @@
     else:
         if count < 0:
             raise oefmt(space.w_ValueError, "negative count")
-        if encoding is not None or errors is not None:
-            raise oefmt(space.w_TypeError,
-                        "encoding or errors without string argument")
         return '\0' * count
-    # Unicode with encoding
-    if space.isinstance_w(w_source, space.w_unicode):
-        if encoding is None:
-            raise RuntimeError
-            raise oefmt(space.w_TypeError,
-                        "string argument without an encoding")
-        from pypy.objspace.std.unicodeobject import encode_object
-        w_source = encode_object(space, w_source, encoding, errors)
-        # and continue with the encoded string
 
     return _convert_from_buffer_or_iterable(space, w_source)
 
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -580,3 +580,7 @@
             assert not b.endswith(bb)
             assert not b.endswith((bb, bb))
             assert bytearray.maketrans(bb, bb)
+
+    def test_constructor_typeerror(self):
+        raises(TypeError, bytearray, b'', 'ascii')
+        raises(TypeError, bytearray, '')
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -934,3 +934,7 @@
         e = raises(TypeError, b'abc'.__getitem__, b'd')
         assert str(e.value).startswith(
             'byte indices must be integers or slices')
+
+    def test_constructor_typeerror(self):
+        raises(TypeError, bytes, b'', 'ascii')
+        raises(TypeError, bytes, '')

From pypy.commits at gmail.com  Fri Nov 18 14:18:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 11:18:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Disable clock_gettime() on OS/X. It's a
 mess because it was only added
Message-ID: <582f541b.42212e0a.17fd5.9ccb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88475:91e202bbd0b9
Date: 2016-11-18 20:18 +0100
http://bitbucket.org/pypy/pypy/changeset/91e202bbd0b9/

Log:	Disable clock_gettime() on OS/X. It's a mess because it was only
	added in 10.12, and there are strange effects when compiling on a
	10.11---it seems the macro CLOCK_MONOTONIC is found anyway?

diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -173,6 +173,12 @@
     state = State()
 
 HAS_CLOCK_GETTIME = (CLOCK_MONOTONIC is not None)
+if sys.platform == 'darwin':
+    HAS_CLOCK_GETTIME = False
+    # ^^^ https://bitbucket.org/pypy/pypy/issues/2432 and others
+    # (change it manually if you *know* you want to build and run on
+    # OS/X 10.12 or later)
+
 if HAS_CLOCK_GETTIME:
     # Linux and other POSIX systems with clock_gettime()
     # TIMESPEC:

From pypy.commits at gmail.com  Fri Nov 18 14:31:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 18 Nov 2016 11:31:02 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix HAVE_FSTATVFS,
 which used to be always False because of a missing
Message-ID: <582f56f6.42212e0a.17fd5.9d84@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88476:09166f2f6bfd
Date: 2016-11-18 20:30 +0100
http://bitbucket.org/pypy/pypy/changeset/09166f2f6bfd/

Log:	Fix HAVE_FSTATVFS, which used to be always False because of a
	missing header in the test

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1874,7 +1874,9 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = eci
+    _compilation_info_ = eci.merge(ExternalCompilationInfo(
+        includes=['sys/statvfs.h'],
+    ))
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown

From pypy.commits at gmail.com  Fri Nov 18 17:26:53 2016
From: pypy.commits at gmail.com (sbauman)
Date: Fri, 18 Nov 2016 14:26:53 -0800 (PST)
Subject: [pypy-commit] pypy value-classes: Merge with default
Message-ID: <582f802d.83092e0a.8c002.a808@mx.google.com>

Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: value-classes
Changeset: r88477:7be1703de735
Date: 2016-11-18 17:26 -0500
http://bitbucket.org/pypy/pypy/changeset/7be1703de735/

Log:	Merge with default

diff too long, truncating to 2000 out of 34881 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3:
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,421 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:

From pypy.commits at gmail.com  Sat Nov 19 11:07:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 08:07:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix for
 __pypy__.test.test_special.test_hidden_applevel_frame: hidden
Message-ID: <583078be.8da2190a.507c.d1e8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88478:8f84fd21ab98
Date: 2016-11-19 17:06 +0100
http://bitbucket.org/pypy/pypy/changeset/8f84fd21ab98/

Log:	Fix for __pypy__.test.test_special.test_hidden_applevel_frame:
	hidden frames should simply not record their exception in
	sys_exc_info.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -757,7 +757,8 @@
         block = SysExcInfoRestorer(last_exception, self.lastblock)
         self.lastblock = block
         if operationerr is not None:   # otherwise, don't change sys_exc_info
-            ec.set_sys_exc_info(operationerr)
+            if not self.hide():
+                ec.set_sys_exc_info(operationerr)
 
     def end_finally(self):
         # unlike CPython, there are two statically distinct cases: the

From pypy.commits at gmail.com  Sat Nov 19 11:40:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 08:40:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix for setting f_lineno
Message-ID: <5830806f.42452e0a.f2fec.d774@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88479:038ed5ec2571
Date: 2016-11-19 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/038ed5ec2571/

Log:	Test and fix for setting f_lineno

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -647,51 +647,43 @@
             raise oefmt(space.w_ValueError,
                         "can't jump to 'except' line as there's no exception")
 
-        # Don't jump into or out of a finally block.
-        # Unlike CPython, we can't jump into or out of an except block
-        # either---there would be a mess with SysExcInfoRestorer.
+        # Don't jump inside or out of an except or a finally block.
+        # Note that CPython doesn't check except blocks,
+        # but that results in crashes (tested on 3.5.2+).
         f_lasti_handler_addr = -1
         new_lasti_handler_addr = -1
-        blockstack = [-1]    # list of odd length:
-                             #   * addr of most recent outermost handler
-                             # [ * addr of start of outermost block
-                             #   * addr of most recent handler in that block 
-                             #       (last two items repeated) ]
+        blockstack = []     # current blockstack (addresses of SETUP_*)
+        endblock = [-1]     # current finally/except block stack
         addr = 0
         while addr < len(code):
             op = ord(code[addr])
             if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
                       SETUP_ASYNC_WITH):
                 blockstack.append(addr)
-                blockstack.append(-1)
             elif op == POP_BLOCK:
-                if len(blockstack) < 3:
+                if len(blockstack) == 0:
                     raise oefmt(space.w_SystemError,
-                                "blocks not properly nested in this bytecode")
-                blockstack.pop()
+                           "POP_BLOCK not properly nested in this bytecode")
                 setup_op = ord(code[blockstack.pop()])
                 if setup_op != SETUP_LOOP:
-                    blockstack[-1] = addr
-            elif op == END_FINALLY:    # "async for" nests blocks
-                blockstack[-1] = -1    # strangely, careful here
+                    endblock.append(addr)
+            elif op == END_FINALLY:
+                if len(endblock) <= 1:
+                    raise oefmt(space.w_SystemError,
+                           "END_FINALLY not properly nested in this bytecode")
+                endblock.pop()
 
-            if addr == new_lasti or addr == self.last_instr:
-                ii = len(blockstack) - 1
-                while ii > 0 and blockstack[ii] == -1:
-                    ii -= 2
-                assert ii >= 0
-                handler_addr = blockstack[ii]
-                if addr == new_lasti:
-                    new_lasti_handler_addr = handler_addr
-                if addr == self.last_instr:
-                    f_lasti_handler_addr = handler_addr
+            if addr == new_lasti:
+                new_lasti_handler_addr = endblock[-1]
+            if addr == self.last_instr:
+                f_lasti_handler_addr = endblock[-1]
 
             if op >= HAVE_ARGUMENT:
                 addr += 3
             else:
                 addr += 1
 
-        if len(blockstack) != 1:
+        if len(blockstack) != 0 or len(endblock) != 1:
             raise oefmt(space.w_SystemError,
                         "blocks not properly nested in this bytecode")
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -121,6 +121,30 @@
         assert str(errors[0]).startswith(
             "can't jump into or out of an 'expect' or 'finally' block")
 
+    def test_f_lineno_set_3(self):
+        def jump_in_nested_finally(output):
+            try:
+                output.append(2)
+            finally:
+                output.append(4)
+                try:
+                    output.append(6)
+                finally:
+                    output.append(8)
+                output.append(9)
+        output = []
+
+        def tracer(f, event, *args):
+            if event == 'line' and len(output) == 1:
+                f.f_lineno += 5
+            return tracer
+
+        import sys
+        sys.settrace(tracer)
+        jump_in_nested_finally(output)
+        sys.settrace(None)
+        assert output == [2, 9]
+
     def test_f_lineno_set_firstline(self):
         r"""
         seen = []

From pypy.commits at gmail.com  Sat Nov 19 11:57:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 08:57:41 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix sys.exc_info in _continuation: we now
 need to save and restore it carefully
Message-ID: <58308485.84c3190a.4cc1b.d96d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88480:6664bdccb98b
Date: 2016-11-19 17:57 +0100
http://bitbucket.org/pypy/pypy/changeset/6664bdccb98b/

Log:	Fix sys.exc_info in _continuation: we now need to save and restore
	it carefully

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -42,6 +42,7 @@
         bottomframe.locals_cells_stack_w[3] = w_kwds
         bottomframe.last_exception = get_cleared_operation_error(space)
         self.bottomframe = bottomframe
+        self.saved_exception = None
         #
         global_state.origin = self
         self.sthread = sthread
@@ -227,6 +228,8 @@
 def new_stacklet_callback(h, arg):
     self = global_state.origin
     self.h = h
+    self.saved_exception = self.sthread.ec.sys_exc_info()
+    self.sthread.ec.set_sys_exc_info(None)
     global_state.clear()
     try:
         frame = self.bottomframe
@@ -248,9 +251,12 @@
     self.h, origin.h = origin.h, h
     #
     current = sthread.ec.topframeref
+    saved_exc = sthread.ec.sys_exc_info()
     sthread.ec.topframeref = self.bottomframe.f_backref
+    sthread.ec.set_sys_exc_info(self.saved_exception)
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
+    origin.saved_exception = saved_exc
     #
     return get_result()
 

From pypy.commits at gmail.com  Sat Nov 19 12:07:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 09:07:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: this case is optimized as well,
 now that we don't need to search the
Message-ID: <583086b4.07452e0a.8034a.d69e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88481:a17f5a7b79d0
Date: 2016-11-19 18:06 +0100
http://bitbucket.org/pypy/pypy/changeset/a17f5a7b79d0/

Log:	this case is optimized as well, now that we don't need to search the
	frame chain.

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -908,12 +908,15 @@
     def test_call_in_subfunction(self):
         import sys
         def g():
-            # this case is not optimized, because we need to search the
-            # frame chain.  it's probably not worth the complications
+            # new in PyPy3: this case is optimized as well, now that we
+            # don't need to search the frame chain.  This can be
+            # regarded as an advantage of the PyPy3 sys_exc_info stored
+            # on the ExecutionContext: although it forces the exception
+            # itself more often, it forces frames less often.
             return sys.exc_info()[1]
         e = KeyError("boom")
         try:
             raise e
         except:
             assert g() is e
-    test_call_in_subfunction.expected = 'n'
+    test_call_in_subfunction.expected = 'y'

From pypy.commits at gmail.com  Sat Nov 19 12:17:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 09:17:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix to not depend on the current working
 directory
Message-ID: <5830893a.57d7190a.6a656.db44@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88482:7813ce03ef1f
Date: 2016-11-19 18:15 +0100
http://bitbucket.org/pypy/pypy/changeset/7813ce03ef1f/

Log:	Fix to not depend on the current working directory

diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -11,11 +11,12 @@
     # similar to testdir.inline_run(), but running a subprocess to avoid
     # confusion.  Parses the standard output of test_all.py, so somehow
     # dependent on how it looks like.
+    cur_dir = str(py.path.local(__file__).dirpath())
     test_all = py.path.local(__file__).dirpath('..', '..', '..', 'test_all.py')
     args = [sys.executable, str(test_all), "-v"] + map(str, args)
     print '=========>', args
     passed, failed = [], []
-    popen = subprocess.Popen(args, stdout=subprocess.PIPE)
+    popen = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cur_dir)
     output, _ = popen.communicate()
 
     for line in output.splitlines(False):

From pypy.commits at gmail.com  Sat Nov 19 13:18:46 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 19 Nov 2016 10:18:46 -0800 (PST)
Subject: [pypy-commit] pypy default: kill unused rtyper.order
Message-ID: <58309786.d52b190a.c9890.db1e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88483:542388518260
Date: 2016-11-19 18:18 +0000
http://bitbucket.org/pypy/pypy/changeset/542388518260/

Log:	kill unused rtyper.order

diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -78,7 +78,6 @@
             self.log.info(s)
         except:
             self.seed = 0
-        self.order = None
 
     def getconfig(self):
         return self.annotator.translator.config
@@ -218,15 +217,9 @@
                 r = random.Random(self.seed)
                 r.shuffle(pending)
 
-            if self.order:
-                tracking = self.order(self.annotator, pending)
-            else:
-                tracking = lambda block: None
-
             previous_percentage = 0
             # specialize all blocks in the 'pending' list
             for block in pending:
-                tracking(block)
                 blockcount += 1
                 self.specialize_block(block)
                 self.already_seen[block] = True

From pypy.commits at gmail.com  Sat Nov 19 14:01:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 11:01:09 -0800 (PST)
Subject: [pypy-commit] pypy default: Another case of @jit.call_shortcut:
 tests that are directly "if x is not
Message-ID: <5830a175.0c20190a.3ea2e.db4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88484:61f7052c3e44
Date: 2016-11-19 19:10 +0000
http://bitbucket.org/pypy/pypy/changeset/61f7052c3e44/

Log:	Another case of @jit.call_shortcut: tests that are directly "if x is
	not None: return x" or "if x != 0: return x"

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -613,8 +613,11 @@
             return
         cs = effectinfo.call_shortcut
         ptr_box = op.getarg(1 + cs.argnum)
-        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                       raw=(ptr_box.type == 'i'))
+        if cs.fielddescr is not None:
+            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                           raw=(ptr_box.type == 'i'))
+        else:
+            value_box = ptr_box
         self.replace_op_with(op, ResOperation(cond_call_opnum,
                                               [value_box] + op.getarglist(),
                                               descr=descr))
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -88,6 +88,13 @@
         call_shortcut_descr = get_call_descr(self.gc_ll_descr,
             [lltype.Ptr(T)], lltype.Signed,
             effectinfo)
+        call_shortcut_2 = CallShortcut(0, None)
+        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut_2)
+        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
+            [lltype.Signed], lltype.Signed,
+            effectinfo_2)
         #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
@@ -1451,3 +1458,14 @@
             i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
             jump(i1)
         """)
+
+    def test_handle_call_shortcut_2(self):
+        self.check_rewrite("""
+            [i0]
+            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """, """
+            [i0]
+            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """)
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -388,15 +388,23 @@
                    return y.field          if r: return r
         """
         block = graph.startblock
-        if len(block.operations) == 0:
-            return
-        op = block.operations[0]
-        if op.opname != 'getfield':
-            return
-        [v_inst, c_fieldname] = op.args
+        operations = block.operations
+        c_fieldname = None
+        if not operations:
+            v_inst = v_result = block.exitswitch
+        else:
+            op = operations[0]
+            if len(op.args) == 0:
+                return
+            if op.opname != 'getfield':  # check for this form:
+                v_inst = op.args[0]      #     if y is not None;
+                v_result = v_inst        #          return y
+            else:
+                operations = operations[1:]
+                [v_inst, c_fieldname] = op.args
+                v_result = op.result
         if not isinstance(v_inst, Variable):
             return
-        v_result = op.result
         if v_result.concretetype != graph.getreturnvar().concretetype:
             return
         if v_result.concretetype == lltype.Void:
@@ -409,7 +417,7 @@
         PSTRUCT = v_inst.concretetype
         v_check = v_result
         fastcase = True
-        for op in block.operations[1:]:
+        for op in operations:
             if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
                     and v_check is op.args[0]):
                 v_check = op.result
@@ -439,5 +447,8 @@
         if not link.target.is_final_block():
             return
 
-        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        if c_fieldname is not None:
+            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        else:
+            fielddescr = None
         return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -424,6 +424,21 @@
             r = b.foobar = 123
         return r
 
+    def f6(b):
+        if b is not None:
+            return b
+        return B()
+
+    def f7(c, a):
+        if a:
+            return a
+        return 123
+
+    def b_or_none(c):
+        if c > 15:
+            return B()
+        return None
+
     def f(a, c):
         b = B()
         f1(a, b, c)
@@ -432,6 +447,8 @@
         r = lltype.malloc(RAW, flavor='raw')
         f4(r)
         f5(b)
+        f6(b_or_none(c))
+        f7(c, a)
 
     rtyper = support.annotate(f, [10, 20])
     f1_graph = rtyper.annotator.translator._graphof(f1)
@@ -444,6 +461,10 @@
     assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
     f5_graph = rtyper.annotator.translator._graphof(f5)
     assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+    f6_graph = rtyper.annotator.translator._graphof(f6)
+    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
+    f7_graph = rtyper.annotator.translator._graphof(f7)
+    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
 
 def test_cant_find_call_shortcut():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU

From pypy.commits at gmail.com  Sat Nov 19 15:19:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 12:19:41 -0800 (PST)
Subject: [pypy-commit] pypy default: Reapply fix c6f52c21fe7e
Message-ID: <5830b3dd.90202e0a.de81b.e103@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88485:baf04e3cc2f7
Date: 2016-11-19 21:17 +0100
http://bitbucket.org/pypy/pypy/changeset/baf04e3cc2f7/

Log:	Reapply fix c6f52c21fe7e

diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
--- a/_pytest/_code/source.py
+++ b/_pytest/_code/source.py
@@ -416,6 +416,8 @@
             trysource = source[start:end]
             if trysource.isparseable():
                 return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
     raise SyntaxError("no valid source range around line %d " % (lineno,))
 
 

From pypy.commits at gmail.com  Sat Nov 19 15:19:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 12:19:43 -0800 (PST)
Subject: [pypy-commit] pypy default: Reapply fix 0867fc01e680
Message-ID: <5830b3df.a118190a.64378.dc99@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88486:45736179ab7e
Date: 2016-11-19 21:18 +0100
http://bitbucket.org/pypy/pypy/changeset/45736179ab7e/

Log:	Reapply fix 0867fc01e680

diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
--- a/_pytest/_code/code.py
+++ b/_pytest/_code/code.py
@@ -601,7 +601,7 @@
 class TerminalRepr:
     def __str__(self):
         s = self.__unicode__()
-        if sys.version_info[0] < 3:
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
             s = s.encode('utf-8')
         return s
 

From pypy.commits at gmail.com  Sat Nov 19 15:38:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 12:38:48 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: next cpython bug
Message-ID: <5830b858.0c20190a.3ea2e.df38@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5749:2c56de222716
Date: 2016-11-19 21:38 +0100
http://bitbucket.org/pypy/extradoc/changeset/2c56de222716/

Log:	next cpython bug

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -121,3 +121,26 @@
   TypeError).  That's a bit confusing and, as far as I can tell, always
   pointless.  Also, related: d.keys()==d.keys() but
   d.values()!=d.values().
+
+* this program fails the check for no sys.exc_info(), even though at
+  the point this assert runs (called from the <== line) we are not in
+  any except/finally block.  This is a generalization of
+  test_exceptions:test_generator_doesnt_retain_old_exc::
+
+    import sys
+
+    def g():
+        try:
+            raise ValueError
+        except ValueError:
+            yield 1
+        assert sys.exc_info() == (None, None, None)
+        yield 2
+
+    gen = g()
+
+    try:
+        raise IndexError
+    except IndexError:
+        assert next(gen) is 1
+    assert next(gen) is 2    # <==

From pypy.commits at gmail.com  Sat Nov 19 16:10:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 13:10:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix one test case. Because we're closely
 following CPython, we have
Message-ID: <5830bfd9.1a4c2e0a.ca6c7.e21d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88487:dc9cd14f5cd4
Date: 2016-11-19 22:10 +0100
http://bitbucket.org/pypy/pypy/changeset/dc9cd14f5cd4/

Log:	Fix one test case. Because we're closely following CPython, we have
	the same bug shown by another (skipped) test.

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -105,6 +105,7 @@
         current_exc_info = ec.sys_exc_info()
         if self.saved_operr is not None:
             ec.set_sys_exc_info(self.saved_operr)
+            self.saved_operr = None
         self.running = True
         try:
             w_result = frame.execute_frame(self, w_arg_or_err)
@@ -119,7 +120,11 @@
         finally:
             frame.f_backref = jit.vref_None
             self.running = False
-            self.saved_operr = ec.sys_exc_info()
+            # note: this is not perfectly correct: see
+            # test_exc_info_in_generator_4.  But it's simpler and
+            # bug-to-bug compatible with CPython 3.5.
+            if frame._any_except_or_finally_handler():
+                self.saved_operr = ec.sys_exc_info()
             ec.set_sys_exc_info(current_exc_info)
         return w_result
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -790,6 +790,16 @@
             assert w_unroller is not None
             return w_unroller
 
+    @jit.unroll_safe
+    def _any_except_or_finally_handler(self):
+        block = self.lastblock
+        while block is not None:
+            if isinstance(block, SysExcInfoRestorer):
+                import pdb; pdb.set_trace()
+                return True
+            block = block.previous
+        return False
+
     def LOAD_BUILD_CLASS(self, oparg, next_instr):
         w_build_class = self.get_builtin().getdictvalue(
             self.space, '__build_class__')
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -497,6 +497,40 @@
             assert next(gen) is LookupError
             assert next(gen) is ValueError
 
+    def test_exc_info_in_generator_3(self):
+        import sys
+        def g():
+            yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+        gen = g()
+        try:
+            raise IndexError
+        except IndexError:
+            assert next(gen) is IndexError
+        assert next(gen) is None
+        try:
+            raise ValueError
+        except ValueError:
+            assert next(gen) is ValueError
+
+    def test_exc_info_in_generator_4(self):
+        skip("buggy behavior, both in CPython and in PyPy")
+        import sys
+        def g():
+            try:
+                raise ValueError
+            except ValueError:
+                yield 1
+            assert sys.exc_info() == (None, None, None)
+            yield 2
+        gen = g()
+        try:
+            raise IndexError
+        except IndexError:
+            assert next(gen) is 1
+        assert next(gen) is 2
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline

From pypy.commits at gmail.com  Sat Nov 19 16:12:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 19 Nov 2016 13:12:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: oups
Message-ID: <5830c044.17052e0a.75161.e2da@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88488:9866199479d1
Date: 2016-11-19 22:11 +0100
http://bitbucket.org/pypy/pypy/changeset/9866199479d1/

Log:	oups

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -795,7 +795,6 @@
         block = self.lastblock
         while block is not None:
             if isinstance(block, SysExcInfoRestorer):
-                import pdb; pdb.set_trace()
                 return True
             block = block.previous
         return False

From pypy.commits at gmail.com  Sat Nov 19 23:35:31 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 19 Nov 2016 20:35:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-merge-PyType_Ready: Close branch
 py3.5-merge-PyType_Ready
Message-ID: <58312813.47c9190a.6d4ad.f20b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5-merge-PyType_Ready
Changeset: r88489:d8379f57ed42
Date: 2016-11-20 04:35 +0000
http://bitbucket.org/pypy/pypy/changeset/d8379f57ed42/

Log:	Close branch py3.5-merge-PyType_Ready


From pypy.commits at gmail.com  Sat Nov 19 23:35:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 19 Nov 2016 20:35:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Merged in py3.5-merge-PyType_Ready (pull
 request #497)
Message-ID: <58312821.d5472e0a.73fb1.eced@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88490:b8eeb81927d2
Date: 2016-11-20 04:35 +0000
http://bitbucket.org/pypy/pypy/changeset/b8eeb81927d2/

Log:	Merged in py3.5-merge-PyType_Ready (pull request #497)

	merge relevant parts of 9a8f9cff8c5d

diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -29,11 +29,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_length = mp_length;
-                tp_as_mapping.mp_subscript = mp_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
@@ -44,8 +39,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue = '''
             static PyObject *
             mp_subscript(PyObject *self, PyObject *key)
             {
@@ -61,6 +55,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_length = mp_length;
+                tp_as_mapping.mp_subscript = mp_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj["hi there"] == 42
@@ -76,11 +76,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_length = sq_length;
-                tp_as_sequence.sq_item = sq_item;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''),
@@ -90,8 +85,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue='''
             static PyObject *
             sq_item(PyObject *self, Py_ssize_t size)
             {
@@ -107,6 +101,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_length = sq_length;
+                tp_as_sequence.sq_item = sq_item;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj[1] == 42
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -235,9 +235,9 @@
         if cls.runappdirect:
             try:
                 import numpy
-                cls.w_numpy_include = [numpy.get_include()]
-            except:
+            except ImportError:
                 skip('numpy not importable')
+            cls.w_numpy_include = [numpy.get_include()]
         else:
             numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
@@ -308,9 +308,6 @@
                 ),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifdef PYPY_VERSION
@@ -353,9 +350,6 @@
                 '''),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 ''',
@@ -411,9 +405,6 @@
                 """),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifndef PYPY_VERSION
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -637,15 +637,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
@@ -660,6 +655,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
@@ -671,14 +671,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_contains = sq_contains;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             sq_contains(PyObject *self, PyObject *value)
             {
@@ -689,6 +685,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         res = "foo" in obj
@@ -826,11 +827,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type.tp_as_number = &intlike_as_number;
-                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
-                intlike_as_number.nb_add = intlike_nb_add;
-                intlike_as_number.nb_power = intlike_nb_pow;
-                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
                 intObj = PyObject_New(IntLikeObject, &IntLike_Type);
                 if (!intObj) {
                     return NULL;
@@ -847,8 +843,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT;
-                if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL;
                 intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp);
                 if (!intObjNoOp) {
                     return NULL;
@@ -856,8 +850,7 @@
 
                 intObjNoOp->ival = intval;
                 return (PyObject *)intObjNoOp;
-             """)], prologue=
-            """
+             """)], prologue="""
             #include <math.h>
             typedef struct
             {
@@ -909,6 +902,14 @@
                 /*tp_name*/             "IntLikeNoOp",
                 /*tp_basicsize*/        sizeof(IntLikeObjectNoOp),
             };
+            """, more_init="""
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT;
+                intlike_as_number.nb_add = intlike_nb_add;
+                intlike_as_number.nb_power = intlike_nb_pow;
+                if (PyType_Ready(&IntLike_Type) < 0) INITERROR;
+                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT;
+                if (PyType_Ready(&IntLike_Type_NoOp) < 0) INITERROR;
             """)
         a = module.newInt(1)
         b = module.newInt(2)
@@ -955,18 +956,11 @@
         else:
             raise AssertionError("did not get TypeError!")
 
-    def test_call_tp_dealloc_when_created_from_python(self):
+    def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
             ("fetchFooType", "METH_VARARGS",
              """
                 PyObject *o;
-                Foo_Type.tp_basicsize = sizeof(FooObject);
-                Foo_Type.tp_dealloc = &dealloc_foo;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
-                Foo_Type.tp_new = &new_foo;
-                Foo_Type.tp_free = &PyObject_Del;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
-
                 o = PyObject_New(PyObject, &Foo_Type);
                 init_foo(o);
                 Py_DECREF(o);   /* calls dealloc_foo immediately */
@@ -1020,6 +1014,13 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            """, more_init="""
+                Foo_Type.tp_basicsize = sizeof(FooObject);
+                Foo_Type.tp_dealloc = &dealloc_foo;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = &new_foo;
+                Foo_Type.tp_free = &PyObject_Del;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             """)
         Foo = module.fetchFooType()
         assert module.getCounter() == 1010
@@ -1061,14 +1062,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_call = &my_tp_call;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyObject *
             my_tp_call(PyObject *self, PyObject *args, PyObject *kwds)
             {
@@ -1078,6 +1075,10 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_call = &my_tp_call;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         x = module.new_obj()
         assert x() == 42
@@ -1087,18 +1088,18 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                FooType_Type.tp_base = &PyType_Type;
-                if (PyType_Ready(&FooType_Type) < 0) return NULL;
                 Py_INCREF(&FooType_Type);
                 return (PyObject *)&FooType_Type;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyTypeObject FooType_Type = {
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.Type",
             };
+            ''', more_init='''
+                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                FooType_Type.tp_base = &PyType_Type;
+                if (PyType_Ready(&FooType_Type) < 0) INITERROR;
             ''')
         FooType = module.getMetaClass()
         if not self.runappdirect:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -241,7 +241,7 @@
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
-    # test_call_tp_dealloc_when_created_from_python, we should not
+    # test_call_tp_dealloc, we should not
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -86,8 +86,11 @@
     def import_extension(self, modname, functions, prologue="",
             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
         body = prologue + make_methods(functions, modname)
-        init = """PyObject *mod = PyModule_Create(&moduledef);"""
+        init = """PyObject *mod = PyModule_Create(&moduledef);
+               """
         if more_init:
+            init += """#define INITERROR return NULL
+                    """
             init += more_init
         init += "\nreturn mod;"
         return self.import_module(

From pypy.commits at gmail.com  Sat Nov 19 23:51:02 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 19 Nov 2016 20:51:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58312bb6.50dd190a.8d16f.3f75@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88491:13526fb40522
Date: 2016-11-20 04:47 +0000
http://bitbucket.org/pypy/pypy/changeset/13526fb40522/

Log:	hg merge default

diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
--- a/_pytest/_code/code.py
+++ b/_pytest/_code/code.py
@@ -601,7 +601,7 @@
 class TerminalRepr:
     def __str__(self):
         s = self.__unicode__()
-        if sys.version_info[0] < 3:
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
             s = s.encode('utf-8')
         return s
 
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
--- a/_pytest/_code/source.py
+++ b/_pytest/_code/source.py
@@ -416,6 +416,8 @@
             trysource = source[start:end]
             if trysource.isparseable():
                 return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
     raise SyntaxError("no valid source range around line %d " % (lineno,))
 
 
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -127,7 +127,9 @@
         setattr(compiler, executable, command)
 
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
+        # compiler_so can be c++ which has no -Wimplicit
+        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
+        compiler.compiler_so.extend(['-O2', '-fPIC'])
         compiler.shared_lib_extension = get_config_var('SO')
         if "CPPFLAGS" in os.environ:
             cppflags = shlex.split(os.environ["CPPFLAGS"])
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -23,3 +23,9 @@
 .. branch: union-side-effects-2
 
 Try to improve the consistency of RPython annotation unions.
+
+.. branch: pytest-2.9.2
+
+.. branch: clean-exported-state
+
+Clean-ups in the jit optimizeopt
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -174,6 +174,12 @@
             return s
             """)
 
+    def teardown_method(self, method):
+        # pytest may keep some objects alive.
+        # So do some clean-up now without waiting for them to die
+        from ..interp_ssl import SOCKET_STORAGE
+        SOCKET_STORAGE._dict.clear()
+
     def test_connect(self):
         import ssl, gc
         ss = ssl.wrap_socket(self.s)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -359,7 +359,10 @@
         size = pybuf.c_len
         ndim = widen(pybuf.c_ndim)
         shape =   [pybuf.c_shape[i]   for i in range(ndim)]
-        strides = [pybuf.c_strides[i] for i in range(ndim)]
+        if pybuf.c_strides:
+            strides = [pybuf.c_strides[i] for i in range(ndim)]
+        else:
+            strides = [1]
         if pybuf.c_format:
             format = rffi.charp2str(pybuf.c_format)
         else:
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -979,8 +979,7 @@
             ("getCounter", "METH_VARARGS",
              """
                 return PyLong_FromLong(foo_counter);
-             """)], prologue=
-            """
+             """)], prologue="""
             typedef struct {
                 PyObject_HEAD
                 int someval[99];
diff --git a/pypy/module/select/conftest.py b/pypy/module/select/conftest.py
deleted file mode 100644
--- a/pypy/module/select/conftest.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import py
-
-def pytest_collect_directory():
-    py.test.importorskip("ctypes")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/pypy/module/test_lib_pypy/ctypes_tests/support.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/support.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/support.py
@@ -2,8 +2,6 @@
 import sys
 import ctypes
 
-py.test.importorskip("ctypes", "1.0.2")
-
 try:
     import _rawffi
 except ImportError:
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
@@ -3,7 +3,10 @@
 import ctypes
 from _ctypes import function
 
-_rawffi = py.test.importorskip("_rawffi")
+try:
+    import _rawffi
+except ImportError:
+    py.test.skip("app-level test only for PyPy")
 
 class TestErrno:
 
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
--- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py
+++ b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
@@ -9,7 +9,9 @@
 
 def pytest_funcarg__child(request):
     try:
-        pexpect = pytest.importorskip('pexpect')
+        import pexpect
+    except ImportError:
+        pytest.skip("no pexpect module")
     except SyntaxError:
         pytest.skip('pexpect wont work on py3k')
     child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
diff --git a/pytest.py b/pytest.py
--- a/pytest.py
+++ b/pytest.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # PYTHON_ARGCOMPLETE_OK
 """
 pytest: unit and functional testing with Python.
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -613,8 +613,11 @@
             return
         cs = effectinfo.call_shortcut
         ptr_box = op.getarg(1 + cs.argnum)
-        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                       raw=(ptr_box.type == 'i'))
+        if cs.fielddescr is not None:
+            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                           raw=(ptr_box.type == 'i'))
+        else:
+            value_box = ptr_box
         self.replace_op_with(op, ResOperation(cond_call_opnum,
                                               [value_box] + op.getarglist(),
                                               descr=descr))
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -88,6 +88,13 @@
         call_shortcut_descr = get_call_descr(self.gc_ll_descr,
             [lltype.Ptr(T)], lltype.Signed,
             effectinfo)
+        call_shortcut_2 = CallShortcut(0, None)
+        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut_2)
+        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
+            [lltype.Signed], lltype.Signed,
+            effectinfo_2)
         #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
@@ -1451,3 +1458,14 @@
             i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
             jump(i1)
         """)
+
+    def test_handle_call_shortcut_2(self):
+        self.check_rewrite("""
+            [i0]
+            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """, """
+            [i0]
+            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """)
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -388,15 +388,23 @@
                    return y.field          if r: return r
         """
         block = graph.startblock
-        if len(block.operations) == 0:
-            return
-        op = block.operations[0]
-        if op.opname != 'getfield':
-            return
-        [v_inst, c_fieldname] = op.args
+        operations = block.operations
+        c_fieldname = None
+        if not operations:
+            v_inst = v_result = block.exitswitch
+        else:
+            op = operations[0]
+            if len(op.args) == 0:
+                return
+            if op.opname != 'getfield':  # check for this form:
+                v_inst = op.args[0]      #     if y is not None;
+                v_result = v_inst        #          return y
+            else:
+                operations = operations[1:]
+                [v_inst, c_fieldname] = op.args
+                v_result = op.result
         if not isinstance(v_inst, Variable):
             return
-        v_result = op.result
         if v_result.concretetype != graph.getreturnvar().concretetype:
             return
         if v_result.concretetype == lltype.Void:
@@ -409,7 +417,7 @@
         PSTRUCT = v_inst.concretetype
         v_check = v_result
         fastcase = True
-        for op in block.operations[1:]:
+        for op in operations:
             if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
                     and v_check is op.args[0]):
                 v_check = op.result
@@ -439,5 +447,8 @@
         if not link.target.is_final_block():
             return
 
-        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        if c_fieldname is not None:
+            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        else:
+            fielddescr = None
         return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -424,6 +424,21 @@
             r = b.foobar = 123
         return r
 
+    def f6(b):
+        if b is not None:
+            return b
+        return B()
+
+    def f7(c, a):
+        if a:
+            return a
+        return 123
+
+    def b_or_none(c):
+        if c > 15:
+            return B()
+        return None
+
     def f(a, c):
         b = B()
         f1(a, b, c)
@@ -432,6 +447,8 @@
         r = lltype.malloc(RAW, flavor='raw')
         f4(r)
         f5(b)
+        f6(b_or_none(c))
+        f7(c, a)
 
     rtyper = support.annotate(f, [10, 20])
     f1_graph = rtyper.annotator.translator._graphof(f1)
@@ -444,6 +461,10 @@
     assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
     f5_graph = rtyper.annotator.translator._graphof(f5)
     assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+    f6_graph = rtyper.annotator.translator._graphof(f6)
+    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
+    f7_graph = rtyper.annotator.translator._graphof(f7)
+    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
 
 def test_cant_find_call_shortcut():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -108,7 +108,7 @@
     """
     log_noopt = False
 
-    def __init__(self, trace, celltoken, state,
+    def __init__(self, trace, celltoken, state, runtime_boxes,
                  call_pure_results=None, enable_opts=None,
                  inline_short_preamble=True):
         self.trace = trace
@@ -117,6 +117,8 @@
         self.state = state
         self.call_pure_results = call_pure_results
         self.inline_short_preamble = inline_short_preamble
+        assert runtime_boxes is not None
+        self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -124,7 +126,11 @@
         assert unroll # we should not be here if it's disabled
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.call_pure_results, self.inline_short_preamble)
+            self.runtime_boxes, self.call_pure_results, self.inline_short_preamble)
+
+    def forget_optimization_info(self):
+        self.state.forget_optimization_info()
+        CompileData.forget_optimization_info(self)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
@@ -290,7 +296,7 @@
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     jitcell_token.target_tokens = [start_descr]
-    loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -360,7 +366,7 @@
     history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
     enable_opts = jitdriver_sd.warmstate.enable_opts
     call_pure_results = metainterp.call_pure_results
-    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, jumpargs,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -372,6 +378,7 @@
         history.cut(cut)
         history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
         loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
+                                     jumpargs,
                                      call_pure_results=call_pure_results,
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
@@ -516,7 +523,7 @@
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
-        #     somehoe for ContinueRunningNormally
+        #     somehow for ContinueRunningNormally
         if reset_values:
             item.reset_value()
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
      AbstractResOp, GuardResOp
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, we_are_debug
 from rpython.jit.metainterp.optimizeopt import info
         
 
@@ -172,7 +172,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getfield(descr, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.AbstractStructPtrInfo):
                 assert opinfo in self.cached_infos
         if isinstance(res, PreambleOp):
@@ -202,7 +202,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getitem(descr, self.index, optheap)
-        if not we_are_translated() and res:
+        if we_are_debug() and res:
             if isinstance(opinfo, info.ArrayPtrInfo):
                 assert opinfo in self.cached_infos
         if (isinstance(res, PreambleOp) and
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,9 +24,20 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
+def check_no_forwarding(lsts):
+    for lst in lsts:
+        for op in lst:
+            assert op.get_forwarded() is None
+
 class LoopInfo(object):
     label_op = None
 
+    def _check_no_forwarding(self):
+        pass
+
+    def forget_optimization_info(self):
+        pass
+
 class BasicLoopInfo(LoopInfo):
     def __init__(self, inputargs, quasi_immutable_deps, jump_op):
         self.inputargs = inputargs
@@ -555,7 +566,8 @@
         return (BasicLoopInfo(trace.inputargs, self.quasi_immutable_deps, last_op),
                 self._newoperations)
 
-    def _clean_optimization_info(self, lst):
+    @staticmethod
+    def _clean_optimization_info(lst):
         for op in lst:
             if op.get_forwarded() is not None:
                 op.set_forwarded(None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -5,6 +5,7 @@
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
+from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 
 class PreambleOp(AbstractResOp):
@@ -18,7 +19,7 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     op = None
-    
+
     def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
@@ -51,7 +52,13 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
-    pass
+    res = None
+
+    def _check_no_forwarding(self):
+        assert self.res.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.res.clear_forwarded()
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -101,6 +108,14 @@
                                        descr=sop.getdescr())
         return ProducedShortOp(self, preamble_op)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.getfield_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.getfield_op.clear_forwarded()
+
     def __repr__(self):
         return "HeapOp(%r)" % (self.res,)
 
@@ -193,6 +208,16 @@
                 l.append(pop)
         return l
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        self.one._check_no_forwarding()
+        self.two._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.one.forget_optimization_info()
+        self.two.forget_optimization_info()
+
     def repr(self, memo):
         return "CompoundOp(%s, %s, %s)" % (self.res.repr(memo),
                                            self.one.repr(memo),
@@ -203,7 +228,7 @@
 
 class ProducedShortOp(AbstractProducedShortOp):
     invented_name = False
-    
+
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
@@ -215,6 +240,14 @@
     def repr(self, memo):
         return self.short_op.repr(memo)
 
+    def _check_no_forwarding(self):
+        self.short_op._check_no_forwarding()
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        self.short_op.forget_optimization_info()
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
 
@@ -235,6 +268,14 @@
     def repr(self, memo):
         return "INP(%s)" % (self.res.repr(memo),)
 
+    def _check_no_forwarding(self):
+        AbstractShortOp._check_no_forwarding(self)
+        assert self.preamble_op.get_forwarded() is None
+
+    def forget_optimization_info(self):
+        AbstractShortOp.forget_optimization_info(self)
+        self.preamble_op.clear_forwarded()
+
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
@@ -454,16 +495,23 @@
         self.sb = sb
         self.extra_same_as = self.sb.extra_same_as
         self.target_token = target_token
+        self.build_inplace = False
 
     def setup(self, jump_args, short, label_args):
         self.jump_args = jump_args
         self.short = short
         self.label_args = label_args
+        self.build_inplace = True
 
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
         """
+        # Could this be considered a speculative error?
+        # This check should only fail when trying to jump to an existing trace
+        # by forcing portions of the virtualstate.
+        if not self.build_inplace:
+            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         op = preamble_op.op.get_box_replacement()
         if preamble_op.invented_name:
             self.extra_same_as.append(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -573,6 +573,7 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
+        state[0]._check_no_forwarding()
         return state
 
     def _convert_call_pure_results(self, d):
@@ -621,7 +622,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
         loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, call_pure_results)
+            celltoken, start_state, runtime_boxes, call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimize import InvalidLoop, SpeculativeError
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo
+     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo, check_no_forwarding
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
@@ -35,7 +35,7 @@
 
     def setinfo_from_preamble_list(self, lst, infos):
         for item in lst:
-            if item is None:
+            if item is None or isinstance(item, Const):
                 continue
             i = infos.get(item, None)
             if i is not None:
@@ -97,7 +97,6 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
-
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
@@ -115,26 +114,22 @@
         return modifier.get_virtual_state(args)
 
     def _check_no_forwarding(self, lsts, check_newops=True):
-        for lst in lsts:
-            for op in lst:
-                assert op.get_forwarded() is None
+        check_no_forwarding(lsts)
         if check_newops:
             assert not self.optimizer._newoperations
 
-
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs,
-                                           runtime_boxes, memo)
+                                           info.inputargs, memo)
         exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state,
+    def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
                              call_pure_results, inline_short_preamble=True):
         trace = trace.get_iter()
         try:
@@ -186,7 +181,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
+                    end_jump, label_op, runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -199,7 +194,7 @@
             # to the preamble.
             try:
                 new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
+                        end_jump, label_op, runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
@@ -276,8 +271,7 @@
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, runtime_boxes,
-                                           box_names_memo)
+                                           info.inputargs, box_names_memo)
         exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
@@ -440,8 +434,7 @@
                 continue
             self._expand_info(item, infos)
 
-    def export_state(self, original_label_args, renamed_inputargs,
-                     runtime_boxes, memo):
+    def export_state(self, original_label_args, renamed_inputargs, memo):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
         self.optimizer.flush()
@@ -462,16 +455,17 @@
             op = produced_op.short_op.res
             if not isinstance(op, Const):
                 self._expand_info(op, infos)
-        self.optimizer._clean_optimization_info(end_args)
         return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
-                             short_inputargs, runtime_boxes, memo)
+                             short_inputargs, memo)
 
     def import_state(self, targetargs, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
                 len(targetargs))
+        self._check_no_forwarding([targetargs])
+        exported_state._check_no_forwarding()
         for i, target in enumerate(exported_state.next_iteration_args):
             source = targetargs[i]
             assert source is not target
@@ -527,13 +521,11 @@
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
     * quasi_immutable_deps - for tracking quasi immutables
-    * runtime_boxes - runtime values for boxes, necessary when generating
-                      guards to jump to
     """
 
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
-                 short_inputargs, runtime_boxes, memo):
+                 short_inputargs, memo):
         self.end_args = end_args
         self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
@@ -541,8 +533,8 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
-        self.runtime_boxes = runtime_boxes
         self.dump(memo)
+        self.forget_optimization_info()
 
     def dump(self, memo):
         if have_debug_prints():
@@ -552,5 +544,35 @@
                 debug_print("  " + box.repr(memo))
             debug_stop("jit-log-exported-state")
 
+    def _check_no_forwarding(self):
+        """ Ensures that no optimization state is attached to relevant operations
+        before importing anything. """
+        # Some of these may be redunant
+        check_no_forwarding([
+            self.end_args,
+            self.next_iteration_args,
+            self.renamed_inputargs,
+            self.short_inputargs,
+            self.exported_infos.keys()])
+        for box in self.short_boxes:
+            box._check_no_forwarding()
+
+    def forget_optimization_info(self):
+        """ Clean up optimization info on all operations stored in the ExportedState.
+
+        This function needs to be called when exporting the optimizer state to
+        prevent leaking of optimization information between invocations of the
+        optimizer.
+
+        That includes cleaning up in the event that optimize_peeled_loop() fails
+        with an InvalidLoop exception, as optimize_peeled_loop() mutates the
+        contents of ExportedState.
+        """
+        Optimizer._clean_optimization_info(self.renamed_inputargs)
+        for box in self.exported_infos.iterkeys():
+            box.clear_forwarded()
+        for box in self.short_boxes:
+            box.forget_optimization_info()
+
     def final(self):
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1976,6 +1976,8 @@
         self.aborted_tracing_greenkey = None
 
     def retrace_needed(self, trace, exported_state):
+        if not we_are_translated():
+            exported_state._check_no_forwarding()
         self.partial_trace = trace
         self.retracing_from = self.potential_retrace_position
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -52,6 +52,10 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
+    def clear_forwarded(self):
+        if self.get_forwarded() is not None:
+            self.set_forwarded(None)
+
     @specialize.arg(1)
     def get_box_replacement(op, not_const=False):
         # Read the chain "op, op._forwarded, op._forwarded._forwarded..."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -295,10 +295,15 @@
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
 _translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
+_rpy_assert_value = CDefinedIntSymbolic('RPY_ASSERT_VALUE', default=1)
 
 def we_are_translated_to_c():
     return we_are_translated() and _translated_to_c
 
+def we_are_debug():
+    """ Returns True when not translated or translated with debugging enabled. """
+    return not we_are_translated() or (_translated_to_c and _rpy_assert_value)
+
 # ____________________________________________________________
 
 def instantiate(cls, nonmovable=False):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1874,7 +1874,9 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = eci
+    _compilation_info_ = eci.merge(ExternalCompilationInfo(
+        includes=['sys/statvfs.h'],
+    ))
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -173,6 +173,12 @@
     state = State()
 
 HAS_CLOCK_GETTIME = (CLOCK_MONOTONIC is not None)
+if sys.platform == 'darwin':
+    HAS_CLOCK_GETTIME = False
+    # ^^^ https://bitbucket.org/pypy/pypy/issues/2432 and others
+    # (change it manually if you *know* you want to build and run on
+    # OS/X 10.12 or later)
+
 if HAS_CLOCK_GETTIME:
     # Linux and other POSIX systems with clock_gettime()
     # TIMESPEC:
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -78,7 +78,6 @@
             self.log.info(s)
         except:
             self.seed = 0
-        self.order = None
 
     def getconfig(self):
         return self.annotator.translator.config
@@ -218,15 +217,9 @@
                 r = random.Random(self.seed)
                 r.shuffle(pending)
 
-            if self.order:
-                tracking = self.order(self.annotator, pending)
-            else:
-                tracking = lambda block: None
-
             previous_percentage = 0
             # specialize all blocks in the 'pending' list
             for block in pending:
-                tracking(block)
                 blockcount += 1
                 self.specialize_block(block)
                 self.already_seen[block] = True
diff --git a/rpython/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py
--- a/rpython/tool/algo/graphlib.py
+++ b/rpython/tool/algo/graphlib.py
@@ -25,18 +25,27 @@
     return edges
 
 def depth_first_search(root, vertices, edges):
-    seen = {}
+    seen = set([root])
     result = []
-    def visit(vertex):
-        result.append(('start', vertex))
-        seen[vertex] = True
-        for edge in edges[vertex]:
-            w = edge.target
-            if w in vertices and w not in seen:
-                visit(w)
-        result.append(('stop', vertex))
-    visit(root)
-    return result
+    stack = []
+    while True:
+        result.append(('start', root))
+        stack.append((root, iter(edges[root])))
+        while True:
+            vertex, iterator = stack[-1]
+            try:
+                edge = next(iterator)
+            except StopIteration:
+                stack.pop()
+                result.append(('stop', vertex))
+                if not stack:
+                    return result
+            else:
+                w = edge.target
+                if w in vertices and w not in seen:
+                    seen.add(w)
+                    root = w
+                    break
 
 def vertices_reachable_from(root, vertices, edges):
     for event, v in depth_first_search(root, vertices, edges):
@@ -97,13 +106,20 @@
             for edge in edges[v]:
                 if edge.target in vertices:
                     edgestack.append(edge)
-                    visit(edge.target)
+                    yield visit(edge.target)
                     edgestack.pop()
             stackpos[v] = None
         else:
             if stackpos[v] is not None:   # back-edge
                 result.append(edgestack[stackpos[v]:])
-    visit(root)
+
+    pending = [visit(root)]
+    while pending:
+        generator = pending[-1]
+        try:
+            pending.append(next(generator))
+        except StopIteration:
+            pending.pop()
     return result        
 
 
@@ -164,14 +180,20 @@
                 raise CycleFound
             if w in unvisited:
                 del unvisited[w]
-                visit(w)
+                yield visit(w)
         del visiting[vertex]
     try:
         unvisited = vertices.copy()
         while unvisited:
             visiting = {}
             root = unvisited.popitem()[0]
-            visit(root)
+            pending = [visit(root)]
+            while pending:
+                generator = pending[-1]
+                try:
+                    pending.append(next(generator))
+                except StopIteration:
+                    pending.pop()
     except CycleFound:
         return False
     else:
diff --git a/rpython/tool/algo/test/test_graphlib.py b/rpython/tool/algo/test/test_graphlib.py
--- a/rpython/tool/algo/test/test_graphlib.py
+++ b/rpython/tool/algo/test/test_graphlib.py
@@ -20,6 +20,22 @@
         'G': [],
         }
 
+    def test_depth_first_search(self):
+        # 'D' missing from the list of vertices
+        lst = depth_first_search('A', list('ABCEFG'), self.edges)
+        assert lst == [
+            ('start', 'A'),
+            ('start', 'B'),
+            ('start', 'E'),
+            ('start', 'C'),
+            ('start', 'F'),
+            ('stop', 'F'),
+            ('stop', 'C'),
+            ('stop', 'E'),
+            ('stop', 'B'),
+            ('stop', 'A'),
+        ]
+
     def test_strong_components(self):
         edges = self.edges
         saved = copy_edges(edges)
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -31,8 +31,10 @@
 RPY_EXTERN
 void RPyAssertFailed(const char* filename, long lineno,
                      const char* function, const char *msg);
+#  define RPY_ASSERT_VALUE 1
 #else
 #  define RPyAssert(x, msg)   /* nothing */
+#  define RPY_ASSERT_VALUE 0
 #endif
 
 RPY_EXTERN

From pypy.commits at gmail.com  Sun Nov 20 10:05:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 07:05:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Messed up the saving/restoring of
 exceptions inside _continuation.
Message-ID: <5831bbbe.944a190a.c33b8.08c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88492:7dc83389f688
Date: 2016-11-20 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/7dc83389f688/

Log:	Messed up the saving/restoring of exceptions inside _continuation.
	Do it hopefully right. Fixes most of
	pypy/module/test_lib_pypy/test_greenlet.

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -42,12 +42,12 @@
         bottomframe.locals_cells_stack_w[3] = w_kwds
         bottomframe.last_exception = get_cleared_operation_error(space)
         self.bottomframe = bottomframe
-        self.saved_exception = None
         #
         global_state.origin = self
         self.sthread = sthread
+        saved_exception = pre_switch(sthread)
         h = sthread.new(new_stacklet_callback)
-        post_switch(sthread, h)
+        post_switch(sthread, h, saved_exception)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -83,8 +83,9 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
+        saved_exception = pre_switch(sthread)
         h = sthread.switch(global_state.destination.h)
-        return post_switch(sthread, h)
+        return post_switch(sthread, h, saved_exception)
 
     @unwrap_spec(w_value = WrappedDefault(None),
                  w_to = WrappedDefault(None))
@@ -228,8 +229,6 @@
 def new_stacklet_callback(h, arg):
     self = global_state.origin
     self.h = h
-    self.saved_exception = self.sthread.ec.sys_exc_info()
-    self.sthread.ec.set_sys_exc_info(None)
     global_state.clear()
     try:
         frame = self.bottomframe
@@ -243,7 +242,12 @@
     global_state.destination = self
     return self.h
 
-def post_switch(sthread, h):
+def pre_switch(sthread):
+    saved_exception = sthread.ec.sys_exc_info()
+    sthread.ec.set_sys_exc_info(None)
+    return saved_exception
+
+def post_switch(sthread, h, saved_exception):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -251,12 +255,10 @@
     self.h, origin.h = origin.h, h
     #
     current = sthread.ec.topframeref
-    saved_exc = sthread.ec.sys_exc_info()
     sthread.ec.topframeref = self.bottomframe.f_backref
-    sthread.ec.set_sys_exc_info(self.saved_exception)
+    sthread.ec.set_sys_exc_info(saved_exception)
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
-    origin.saved_exception = saved_exc
     #
     return get_result()
 
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -708,3 +708,32 @@
 
         continulet.switch(c1, to=c2)
         raises(error, continulet.switch, c1, to=c2)
+
+    def test_exc_info_save_restore(self):
+        from _continuation import continulet
+        import sys
+        main = []
+
+        def f(c):
+            print("in f... 222")
+            try:
+                raise ValueError('fun')
+            except:
+                print("333")
+                exc_info = sys.exc_info()
+                print("444")
+                c17650 = continulet(h)
+                bd50.switch(to=c17650)
+                print("back in f...")
+                assert exc_info == sys.exc_info()
+
+        def h(c):
+            print("in h... 555")
+            assert sys.exc_info() == (None, None, None)
+            print("666")
+
+        main = continulet.__new__(continulet)
+        print(111)
+        bd50 = continulet(f)
+        main.switch(to=bd50)
+        print(999)

From pypy.commits at gmail.com  Sun Nov 20 10:48:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 07:48:43 -0800 (PST)
Subject: [pypy-commit] pypy default: Test and fix for 'methodcaller(...,
 self=...)'
Message-ID: <5831c5db.d7092e0a.c9718.0db3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88493:030861c2e0a2
Date: 2016-11-20 16:48 +0100
http://bitbucket.org/pypy/pypy/changeset/030861c2e0a2/

Log:	Test and fix for 'methodcaller(..., self=...)'

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -130,9 +130,12 @@
 
 
 class methodcaller(object):
-    def __init__(self, method_name, *args, **kwargs):
+    def __init__(*args, **kwargs):
+        if len(args) < 2:
+            raise TypeError("methodcaller() called with not enough arguments")
+        self, method_name = args[:2]
         self._method_name = method_name
-        self._args = args
+        self._args = args[2:]
         self._kwargs = kwargs
 
     def __call__(self, obj):
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -244,6 +244,13 @@
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
+    def test_methodcaller_self(self):
+        from operator import methodcaller
+        class X:
+            def method(myself, self):
+                return self * 6
+        assert methodcaller("method", self=7)(X()) == 42
+
     def test_index(self):
         import operator
         assert operator.index(42) == 42

From pypy.commits at gmail.com  Sun Nov 20 10:49:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 07:49:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5831c625.50202e0a.fd1a8.0d55@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88494:c43509390348
Date: 2016-11-20 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/c43509390348/

Log:	hg merge default

diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -91,11 +91,14 @@
 
 
 class methodcaller(object):
-    def __init__(self, method_name, *args, **kwargs):
+    def __init__(*args, **kwargs):
+        if len(args) < 2:
+            raise TypeError("methodcaller() called with not enough arguments")
+        self, method_name = args[:2]
         if not isinstance(method_name, str):
             raise TypeError("method name must be a string")
         self._method_name = method_name
-        self._args = args
+        self._args = args[2:]
         self._kwargs = kwargs
 
     def __call__(self, obj):
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -182,6 +182,13 @@
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
+    def test_methodcaller_self(self):
+        from operator import methodcaller
+        class X:
+            def method(myself, self):
+                return self * 6
+        assert methodcaller("method", self=7)(X()) == 42
+
     def test_methodcaller_not_string(self):
         import _operator as operator
         e = raises(TypeError, operator.methodcaller, 42)

From pypy.commits at gmail.com  Sun Nov 20 10:55:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 07:55:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Using check_impl_detail() like this is
 not following the spirit of
Message-ID: <5831c771.87a3190a.b9a92.10c8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88495:818fb73d805d
Date: 2016-11-20 16:54 +0100
http://bitbucket.org/pypy/pypy/changeset/818fb73d805d/

Log:	Using check_impl_detail() like this is not following the spirit of
	check_impl_detail(), which should be: "if false, avoid checking some
	detail", not "if false, check that the value is really different".

	Removed anyway, as it seems pypy gives the same answer as cpython
	nowadays.

diff --git a/lib-python/3/test/test_index.py b/lib-python/3/test/test_index.py
--- a/lib-python/3/test/test_index.py
+++ b/lib-python/3/test/test_index.py
@@ -66,10 +66,7 @@
         direct_index = my_int.__index__()
         operator_index = operator.index(my_int)
         self.assertEqual(direct_index, 8)
-        if support.check_impl_detail():
-            self.assertEqual(operator_index, 7)
-        else:
-            self.assertEqual(operator_index, 8)
+        self.assertEqual(operator_index, 7)
         # Both results should be of exact type int.
         self.assertIs(type(direct_index), int)
         #self.assertIs(type(operator_index), int)
@@ -90,10 +87,7 @@
 
         bad_int = BadInt2()
         n = operator.index(bad_int)
-        if support.check_impl_detail():
-            self.assertEqual(n, 0)
-        else:
-            self.assertEqual(n, 1)
+        self.assertEqual(n, 0)
 
 
 class SeqTestCase:

From pypy.commits at gmail.com  Sun Nov 20 11:16:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 08:16:29 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Another CPython 3.5 failure
Message-ID: <5831cc5d.5043190a.885dc.1591@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5750:b6cd4ef2e6cc
Date: 2016-11-20 17:16 +0100
http://bitbucket.org/pypy/extradoc/changeset/b6cd4ef2e6cc/

Log:	Another CPython 3.5 failure

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -144,3 +144,38 @@
     except IndexError:
         assert next(gen) is 1
     assert next(gen) is 2    # <==
+
+* frame.clear() does not clear f_locals, unlike what a test says
+  (Lib/test/test_frame.py)::
+
+    def test_locals_clear_locals(self):
+        # Test f_locals before and after clear() (to exercise caching)
+        f, outer, inner = self.make_frames()
+        outer.f_locals
+        inner.f_locals
+        outer.clear()
+        inner.clear()
+        self.assertEqual(outer.f_locals, {})
+        self.assertEqual(inner.f_locals, {})
+
+  This test passes, but the C-level PyFrameObject has got a strong
+  reference to f_locals, which is only updated (to be empty) if the
+  Python code tries to read this attribute.  In the normal case,
+  code that calls clear() but doesn't read f_locals afterwards will
+  still leak everything contained in the C-level f_locals field.  This
+  can be shown by this failing test::
+
+    import sys
+
+    def g():
+        x = 42
+        return sys._getframe()
+
+    frame = g()
+    d = frame.f_locals
+    frame.clear()
+    print(d)
+    assert d == {}   # fails!  but 'assert d is frame.f_locals' passes,
+                     # which shows that this dict is kept alive by
+                     # 'frame'; and we've seen that it is non-empty
+                     # as long as we don't read frame.f_locals.

From pypy.commits at gmail.com  Sun Nov 20 11:26:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 08:26:48 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix for frame.clear()
Message-ID: <5831cec8.04c7190a.8c86e.140f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88496:e7e2aa356442
Date: 2016-11-20 17:26 +0100
http://bitbucket.org/pypy/pypy/changeset/e7e2aa356442/

Log:	Test and fix for frame.clear()

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -782,8 +782,10 @@
 
     def descr_clear(self, space):
         """F.clear(): clear most references held by the frame"""
-        # Clears a random subset of the attributes (e.g. the fast
-        # locals, but not f_locals).
+        # Clears a random subset of the attributes: the local variables
+        # and the w_locals.  Note that CPython doesn't clear f_locals
+        # (which can create leaks) but it's hard to notice because
+        # the next Python-level read of 'frame.f_locals' will clear it.
         if not self.frame_finished_execution:
             if not self._is_generator_or_coroutine():
                 raise oefmt(space.w_RuntimeError,
@@ -800,6 +802,8 @@
         debug = self.getdebug()
         if debug is not None:
             debug.w_f_trace = None
+            if debug.w_locals is not None:
+                debug.w_locals = space.newdict()
 
         # clear the locals, including the cell/free vars, and the stack
         for i in range(len(self.locals_cells_stack_w)):
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -618,6 +618,15 @@
         #
         raises(StopIteration, next, gen)
 
+    def test_frame_clear_really(self):
+        import sys
+        def f(x):
+            return sys._getframe()
+        frame = f(42)
+        assert frame.f_locals['x'] == 42
+        frame.clear()
+        assert frame.f_locals == {}
+
     def test_throw_trace_bug(self):
         import sys
         def f():

From pypy.commits at gmail.com  Sun Nov 20 11:34:45 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 08:34:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: deque.__contains__()
Message-ID: <5831d0a5.15c1190a.51236.11b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88497:4ab8b2a8f971
Date: 2016-11-20 17:34 +0100
http://bitbucket.org/pypy/pypy/changeset/4ab8b2a8f971/

Log:	deque.__contains__()

diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -284,8 +284,7 @@
         self.modified()
         return w_obj
 
-    def remove(self, w_x):
-        "Remove first occurrence of value."
+    def _find(self, w_x):
         space = self.space
         block = self.leftblock
         index = self.leftindex
@@ -295,14 +294,25 @@
             equal = space.eq_w(w_item, w_x)
             self.checklock(lock)
             if equal:
-                self.del_item(i)
-                return
+                return i
             # Advance the block/index pair
             index += 1
             if index >= BLOCKLEN:
                 block = block.rightlink
                 index = 0
-        raise oefmt(space.w_ValueError, "deque.remove(x): x not in deque")
+        return -1
+
+    def remove(self, w_x):
+        "Remove first occurrence of value."
+        i = self._find(w_x)
+        if i < 0:
+            raise oefmt(self.space.w_ValueError,
+                        "deque.remove(x): x not in deque")
+        self.del_item(i)
+
+    def contains(self, w_x):
+        i = self._find(w_x)
+        return self.space.newbool(i >= 0)
 
     def reverse(self):
         "Reverse *IN PLACE*."
@@ -582,6 +592,7 @@
     __imul__ = interp2app(W_Deque.imul),
     __rmul__ = interp2app(W_Deque.rmul),
     maxlen = GetSetProperty(W_Deque.get_maxlen),
+    __contains__ = interp2app(W_Deque.contains),
 )
 
 # ------------------------------------------------------------
diff --git a/pypy/module/_collections/test/test_deque.py b/pypy/module/_collections/test/test_deque.py
--- a/pypy/module/_collections/test/test_deque.py
+++ b/pypy/module/_collections/test/test_deque.py
@@ -364,6 +364,8 @@
             d.insert(i, 'a')
             assert 'a' in d
             assert 'b' not in d
+            assert d.__contains__('a')
+            assert not d.__contains__('b')
             assert d.index('a') == i
         d = deque(range(10))
         d.insert(-1, 500)

From pypy.commits at gmail.com  Sun Nov 20 11:39:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 08:39:42 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Deque addition specifically checks that
 the argument is another deque.
Message-ID: <5831d1ce.84c3190a.4cc1b.1581@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88498:83ba4f51767c
Date: 2016-11-20 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/83ba4f51767c/

Log:	Deque addition specifically checks that the argument is another
	deque.

diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -178,11 +178,12 @@
                 raise
             self.append(w_obj)
 
-    def add(self, w_iterable):
+    def add(self, w_deque):
+        deque = self.space.interp_w(W_Deque, w_deque)
         copy = W_Deque(self.space)
         copy.maxlen = self.maxlen
         copy.extend(self.iter())
-        copy.extend(w_iterable)
+        copy.extend(deque.iter())
         return self.space.wrap(copy)
 
     def iadd(self, w_iterable):
diff --git a/pypy/module/_collections/test/test_deque.py b/pypy/module/_collections/test/test_deque.py
--- a/pypy/module/_collections/test/test_deque.py
+++ b/pypy/module/_collections/test/test_deque.py
@@ -122,6 +122,10 @@
         d2 = deque([3,4,5])
         assert d1 + d2 == deque([1,2,3,3,4,5])
 
+    def test_cannot_add_list(self):
+        from _collections import deque
+        raises(TypeError, "deque([2]) + [3]")
+
     def test_iadd(self):
         from _collections import deque
         d = deque('a')

From pypy.commits at gmail.com  Sun Nov 20 12:01:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 20 Nov 2016 09:01:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix the repr of defaultdicts in case of
 recursion
Message-ID: <5831d6d6.57202e0a.228d2.1434@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88499:9865394f0452
Date: 2016-11-20 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/9865394f0452/

Log:	Fix the repr of defaultdicts in case of recursion

diff --git a/pypy/module/_collections/app_defaultdict.py b/pypy/module/_collections/app_defaultdict.py
--- a/pypy/module/_collections/app_defaultdict.py
+++ b/pypy/module/_collections/app_defaultdict.py
@@ -31,14 +31,16 @@
 
     def __repr__(self, recurse=set()):
         # XXX not thread-safe, but good enough
+        dictrepr = super(defaultdict, self).__repr__()
         if id(self) in recurse:
-            return "defaultdict(...)"
-        try:
-            recurse.add(id(self))
-            return "defaultdict(%s, %s)" % (repr(self.default_factory),
-                                            super(defaultdict, self).__repr__())
-        finally:
-            recurse.remove(id(self))
+            factoryrepr = "..."
+        else:
+            try:
+                recurse.add(id(self))
+                factoryrepr = repr(self.default_factory)
+            finally:
+                recurse.remove(id(self))
+        return "defaultdict(%s, %s)" % (factoryrepr, dictrepr)
 
     def copy(self):
         return type(self)(self.default_factory, self)
diff --git a/pypy/module/_collections/test/test_defaultdict.py b/pypy/module/_collections/test/test_defaultdict.py
--- a/pypy/module/_collections/test/test_defaultdict.py
+++ b/pypy/module/_collections/test/test_defaultdict.py
@@ -90,3 +90,12 @@
         d = _collections.defaultdict(None, {3: 4})
         dict_iter = d.__reduce__()[4]
         assert type(dict_iter) is type(iter(d.items()))
+
+    def test_rec_repr(self):
+        import _collections
+        class X(_collections.defaultdict):
+            def mydefault(self):
+                pass
+        d = X.__new__(X)
+        d.__init__(d.mydefault)
+        assert repr(d).endswith('defaultdict(..., {})>, {})')

From pypy.commits at gmail.com  Sun Nov 20 16:53:51 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 20 Nov 2016 13:53:51 -0800 (PST)
Subject: [pypy-commit] pypy better-storesink: document and cleanup stuff
Message-ID: <58321b6f.8dcd190a.4fa62.242c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r88500:94adb0d43fce
Date: 2016-11-20 22:45 +0100
http://bitbucket.org/pypy/pypy/changeset/94adb0d43fce/

Log:	document and cleanup stuff

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -1,3 +1,9 @@
+""" A very simple common subexpression elimination pass. It's a very simple
+forward pass, that simply eliminates operations that were executed in all paths
+leading to the current block. Information flows strictly forward, using a cache
+of already seen operations. Caches are merged at control flow merges.
+
+No loop invariant code motion occurs (yet). """
 import collections
 
 from rpython.translator.backendopt import support
@@ -40,6 +46,7 @@
             heapcache = {}
         if new_unions is None:
             new_unions = unionfind.UnionFind()
+        # (opname, concretetype of result, args) -> previous (life) result
         self.purecache = purecache
         self.heapcache = heapcache
         self.variable_families = variable_families
@@ -61,6 +68,8 @@
         # the *cache dictionaries, never to actually put any new variable into
         # the graph, because the concretetypes can change when calling
         # _var_rep.
+        if not isinstance(var, Variable):
+            return var
         var = self.new_unions.find_rep(var)
         return self.variable_families.find_rep(var)
 
@@ -71,6 +80,9 @@
         return (opname, concretetype, tuple(listargs))
 
     def _find_new_res(self, results):
+        """ merges a list of results into a new variable. If all the results
+        are the same, just use that, in which case it's not necessary to pass
+        it along any links either. """
         # helper function for _merge_results
         first = self._var_rep(results[0])
         newres = None
@@ -101,17 +113,27 @@
             backedge.args.append(newres)
         return newres
 
-    def merge(self, firstlink, tuples, backedges):
+    def _merge(self, firstlink, tuples, backedges):
+        """ The core algorithm of merging: actually merge many caches. """
         purecache = {}
         block = firstlink.target
-        # copy all operations that exist in *all* blocks over. need to add a new
-        # inputarg if the result is really a variable
+        # copy all operations that exist in *all* blocks over.
 
         # note that a backedge is not a problem for regular pure operations:
         # since the argument is a phi node iff it is not loop invariant,
         # copying things over is always save (yay SSA form!)
 
-        # try non-straight merges
+        # try non-straight merges: they are merges where the operands are
+        # different in the previous blocks, but where the arguments themselves
+        # are merged into a new variable in the target block
+        # this is code like this:
+        # if <cond>
+        #     x = i + 1
+        #     a = i
+        # else:
+        #     y = j + 1
+        #     a = j
+        # here, a + 1 is redundant, and can be replaced by the merge of x and y
         for argindex in range(len(block.inputargs)):
             inputarg = block.inputargs[argindex]
             # bit slow, but probably ok
@@ -136,8 +158,9 @@
                     newres = self._merge_results(tuples, results, backedges)
                     purecache[newkey] = newres
 
+        # the simple case: the operation is really performed on the *same*
+        # operands. This is the case if the key exists in all other caches
         for key, res in self.purecache.iteritems():
-            # "straight" merge: the variable is in all other caches
             results = [res]
             for link, cache in tuples[1:]:
                 val = cache.purecache.get(key, None)
@@ -228,10 +251,7 @@
         self.new_unions.union(res, op.result)
 
     def cse_block(self, block):
-        def representative_arg(arg):
-            if isinstance(arg, Variable):
-                return self._var_rep(arg)
-            return arg
+        """ perform common subexpression elimination on block. """
         added_same_as = 0
         for opindex in range(len(block.operations) - block.canraise):
             op = block.operations[opindex]
@@ -239,7 +259,7 @@
             if op.opname == 'getfield':
                 fieldname = op.args[1].value
                 concretetype = op.args[0].concretetype
-                arg0 = representative_arg(op.args[0])
+                arg0 = self._var_rep(op.args[0])
                 key = (arg0, op.args[0].concretetype, fieldname)
                 res = self.heapcache.get(key, None)
                 if res is not None:
@@ -250,20 +270,20 @@
                 continue
             if op.opname == 'setfield':
                 concretetype = op.args[0].concretetype
-                target = representative_arg(op.args[0])
+                target = self._var_rep(op.args[0])
                 fieldname = op.args[1].value
                 key = (target, concretetype, fieldname)
                 res = self.heapcache.get(key, None)
                 if (res is not None and
-                        representative_arg(res) ==
-                                representative_arg(op.args[2])):
+                        self._var_rep(res) ==
+                                self._var_rep(op.args[2])):
                     # writing the same value that's already there
                     op.opname = "same_as"
                     op.args = [Constant("not needed setfield", lltype.Void)]
                     added_same_as += 1
                     continue
                 self._clear_heapcache_for(concretetype, fieldname)
-                self.heapcache[target, concretetype, fieldname] = op.args[2]
+                self.heapcache[key] = op.args[2]
                 continue
             if op.opname == "jit_force_virtualizable":
                 T = op.args[0].concretetype
@@ -276,7 +296,7 @@
             if op.opname == "malloc_varsize":
                 # we can remember the size of the malloced object
                 key = ("getarraysize", lltype.Signed,
-                       (representative_arg(op.result), ))
+                       (self._var_rep(op.result), ))
                 self.purecache[key] = op.args[2]
 
             can_fold_op = can_fold(op)
@@ -289,12 +309,12 @@
                     # can't hash pointers, so use the graph directly
                     key = ("direct_call", op.result.concretetype,
                            (funcobj.graph, ) +
-                               tuple([representative_arg(arg)
+                               tuple([self._var_rep(arg)
                                    for arg in op.args[1:]]))
                     can_fold_op = True
             elif can_fold_op:
                 key = (op.opname, op.result.concretetype,
-                       tuple([representative_arg(arg) for arg in op.args]))
+                       tuple([self._var_rep(arg) for arg in op.args]))
 
 
             if has_side_effects_op:
@@ -318,21 +338,26 @@
                 self.purecache[key] = op.result
         return added_same_as
 
-def _merge(tuples, variable_families, analyzer, loop_blocks, backedges):
-    if not tuples:
-        return Cache(variable_families, analyzer)
-    if len(tuples) == 1:
-        (link, cache), = tuples
-        result = cache.copy()
-    else:
-        firstlink, firstcache = tuples[0]
-        result = firstcache.merge(firstlink, tuples, backedges)
-    if loop_blocks:
-        # for all blocks in the loop, clean the heapcache for their effects
-        # that way, loop-invariant reads can be removed, if no one writes to
-        # anything that can alias with them.
-        result._clear_heapcache_for_loop_blocks(loop_blocks)
-    return result
+    @staticmethod
+    def merge(tuples, variable_families, analyzer, loop_blocks, backedges):
+        """ merge list of
+        (incoming link, cache in that link's prevblock)
+        tuples into one new cache that holds in the target block.
+        """
+        if not tuples:
+            return Cache(variable_families, analyzer)
+        if len(tuples) == 1:
+            (link, cache), = tuples
+            result = cache.copy()
+        else:
+            firstlink, firstcache = tuples[0]
+            result = firstcache._merge(firstlink, tuples, backedges)
+        if loop_blocks:
+            # for all blocks in the loop, clean the heapcache for their effects
+            # that way, loop-invariant reads can be removed, if no one writes to
+            # anything that can alias with them.
+            result._clear_heapcache_for_loop_blocks(loop_blocks)
+        return result
 
 def compute_reachability_no_backedges(graph, backedges):
     reachable = {}
@@ -374,25 +399,27 @@
         result[block] = loop_blocks
     return result
 
-    loop_blocks = support.find_loop_blocks(graph)
-    result = {}
-    for loop_block, start in loop_blocks.iteritems():
-        result.setdefault(start, []).append(loop_block)
-    return result
-
 class CSE(object):
     def __init__(self, translator):
         self.translator = translator
         self.analyzer = WriteAnalyzer(translator)
 
     def transform(self, graph):
+        """ Perform CSE on graph. """
+        # this optimization really works on SSA graphs. don't transform the
+        # graph, but use the data flow analysis of SSA to figure out which SSI
+        # variables would be the same SSA variable
         variable_families = ssa.DataFlowFamilyBuilder(graph).get_variable_families()
         entrymap = mkentrymap(graph)
         backedges = support.find_backedges(graph)
         loops = loop_blocks(graph, backedges, entrymap)
         todo = collections.deque([graph.startblock])
+
+        # map blocks to a list of
+        # (incoming link, cache in that link's prevblock)
         caches_to_merge = collections.defaultdict(list)
         done = set()
+        enqueued = {graph.startblock}
 
         added_same_as = 0
 
@@ -404,21 +431,24 @@
                                     if link in backedges]
 
             if not block.is_final_block():
-                cache = _merge(
+                cache = Cache.merge(
                     caches_to_merge[block], variable_families, self.analyzer,
                     loops.get(block, None), current_backedges)
                 added_same_as += cache.cse_block(block)
             else:
-                cache = Cache(variable_families, self.analyzer)
+                cache = None
             done.add(block)
-            # add all target blocks where all predecessors are already done
+            # add all target blocks where all predecessors are already done,
+            # or are backedges
             for exit in block.exits:
                 for lnk in entrymap[exit.target]:
                     if lnk.prevblock not in done and lnk not in backedges:
                         break
                 else:
-                    if exit.target not in done and exit.target not in todo: # XXX
+                    if exit.target not in done and exit.target not in enqueued:
                         todo.append(exit.target)
+                        enqueued.add(exit.target)
+                assert cache is not None # final blocks don't have exits
                 caches_to_merge[exit.target].append((exit, cache))
         simplify.transform_dead_op_vars(graph)
         if added_same_as:

From pypy.commits at gmail.com  Sun Nov 20 16:53:53 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 20 Nov 2016 13:53:53 -0800 (PST)
Subject: [pypy-commit] pypy better-storesink: remove the ability to deal
 with elidable calls: this is never useful anywhere
Message-ID: <58321b71.57202e0a.228d2.215c@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r88501:38b30af12c5c
Date: 2016-11-20 22:46 +0100
http://bitbucket.org/pypy/pypy/changeset/38b30af12c5c/

Log:	remove the ability to deal with elidable calls: this is never useful
	anywhere in pypy

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -301,18 +301,7 @@
 
             can_fold_op = can_fold(op)
             has_side_effects_op = has_side_effects(op)
-            if op.opname == "direct_call":
-                funcobj = op.args[0].value._obj
-                func = getattr(funcobj, '_callable', None)
-                elidable = getattr(func, "_elidable_function_", False)
-                if elidable:
-                    # can't hash pointers, so use the graph directly
-                    key = ("direct_call", op.result.concretetype,
-                           (funcobj.graph, ) +
-                               tuple([self._var_rep(arg)
-                                   for arg in op.args[1:]]))
-                    can_fold_op = True
-            elif can_fold_op:
+            if can_fold_op:
                 key = (op.opname, op.result.concretetype,
                        tuple([self._var_rep(arg) for arg in op.args]))
 
diff --git a/rpython/translator/backendopt/test/test_cse.py b/rpython/translator/backendopt/test/test_cse.py
--- a/rpython/translator/backendopt/test/test_cse.py
+++ b/rpython/translator/backendopt/test/test_cse.py
@@ -557,28 +557,6 @@
             return len(l)
         self.check(f, [int], fullopts=True, getarraysize=0)
 
-    def test_remove_duplicate_elidable_call(self):
-        @jit.elidable
-        def p(x):
-            return x + 1
-
-        def f(x):
-            return p(x) + p(x)
-
-        self.check(f, [int], direct_call=1)
-
-    def test_remove_duplicate_elidable_call_raises(self):
-        @jit.elidable
-        def p(x):
-            return x + 1
-
-        def f(x):
-            try:
-                return p(x) + p(x)
-            except IndexError:
-                return -5
-
-        self.check(f, [int], direct_call=2)
 
 def fakevar(name='v'):
     var = Variable(name)

From pypy.commits at gmail.com  Sun Nov 20 16:53:55 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 20 Nov 2016 13:53:55 -0800 (PST)
Subject: [pypy-commit] pypy better-storesink: there's not actually a need
 for a separate union find, just use variable_families
Message-ID: <58321b73.0faa190a.f6d94.2094@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r88502:6d0161df2581
Date: 2016-11-20 22:53 +0100
http://bitbucket.org/pypy/pypy/changeset/6d0161df2581/

Log:	there's not actually a need for a separate union find, just use
	variable_families

diff --git a/rpython/translator/backendopt/cse.py b/rpython/translator/backendopt/cse.py
--- a/rpython/translator/backendopt/cse.py
+++ b/rpython/translator/backendopt/cse.py
@@ -38,24 +38,21 @@
     return getattr(llop, op.opname).canfold
 
 class Cache(object):
-    def __init__(self, variable_families, analyzer, new_unions=None,
+    def __init__(self, variable_families, analyzer,
                  purecache=None, heapcache=None):
         if purecache is None:
             purecache = {}
         if heapcache is None:
             heapcache = {}
-        if new_unions is None:
-            new_unions = unionfind.UnionFind()
         # (opname, concretetype of result, args) -> previous (life) result
         self.purecache = purecache
         self.heapcache = heapcache
         self.variable_families = variable_families
         self.analyzer = analyzer
-        self.new_unions = new_unions
 
     def copy(self):
         return Cache(
-                self.variable_families, self.analyzer, self.new_unions,
+                self.variable_families, self.analyzer,
                 self.purecache.copy(),
                 self.heapcache.copy())
 
@@ -70,7 +67,6 @@
         # _var_rep.
         if not isinstance(var, Variable):
             return var
-        var = self.new_unions.find_rep(var)
         return self.variable_families.find_rep(var)
 
     def _key_with_replacement(self, key, index, var):
@@ -211,7 +207,7 @@
                 newres = self._merge_results(tuples, results, backedges)
                 heapcache[key] = newres
         return Cache(
-                self.variable_families, self.analyzer, self.new_unions,
+                self.variable_families, self.analyzer,
                 purecache, heapcache)
 
     def _clear_heapcache_for(self, concretetype, fieldname):
@@ -248,7 +244,9 @@
         assert op.result.concretetype == res.concretetype
         op.opname = 'same_as'
         op.args = [res]
-        self.new_unions.union(res, op.result)
+        # now that we know that the variables are the same, just merge them in
+        # variable_families too
+        self.variable_families.union(res, op.result)
 
     def cse_block(self, block):
         """ perform common subexpression elimination on block. """
@@ -313,8 +311,9 @@
             if op.opname == "cast_pointer":
                 # cast_pointer is a pretty strange operation! it introduces
                 # more aliases, that confuse the CSE pass. Therefore we unify
-                # the two variables in new_unions, to improve the folding.
-                self.new_unions.union(op.args[0], op.result)
+                # the two variables in variable_families, to improve the
+                # folding.
+                self.variable_families.union(op.args[0], op.result)
                 # don't do anything further
                 continue
             if not can_fold_op:

From pypy.commits at gmail.com  Sun Nov 20 16:55:26 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 20 Nov 2016 13:55:26 -0800 (PST)
Subject: [pypy-commit] pypy better-storesink: merge default
Message-ID: <58321bce.2215190a.58fac.1eac@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r88503:b612e2205d01
Date: 2016-11-20 22:54 +0100
http://bitbucket.org/pypy/pypy/changeset/b612e2205d01/

Log:	merge default

diff too long, truncating to 2000 out of 84636 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py
+	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,423 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)

From pypy.commits at gmail.com  Sun Nov 20 17:01:09 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 20 Nov 2016 14:01:09 -0800 (PST)
Subject: [pypy-commit] pypy better-storesink: remove the previous storesink
 (it's fully replaced by the new CSE pass)
Message-ID: <58321d25.07452e0a.8034a.21f2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: better-storesink
Changeset: r88504:986d2adef586
Date: 2016-11-20 22:59 +0100
http://bitbucket.org/pypy/pypy/changeset/986d2adef586/

Log:	remove the previous storesink (it's fully replaced by the new CSE
	pass)

diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py
deleted file mode 100644
--- a/rpython/translator/backendopt/storesink.py
+++ /dev/null
@@ -1,97 +0,0 @@
-
-from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.flowspace.model import mkentrymap, Variable
-from rpython.translator.backendopt import removenoops
-from rpython.translator import simplify
-
-def has_side_effects(op):
-    if op.opname == 'debug_assert' or op.opname == 'jit_force_virtualizable':
-        return False
-    try:
-        return getattr(llop, op.opname).sideeffects
-    except AttributeError:
-        return True
-
-
-def storesink_graph(graph):
-    """ remove superfluous getfields. use a super-local method: all non-join
-    blocks inherit the heap information from their (single) predecessor
-    """
-    added_some_same_as = False
-    entrymap = mkentrymap(graph)
-
-    # all merge blocks are starting points
-    todo = [(block, None, None) for (block, prev_blocks) in entrymap.iteritems()
-                if len(prev_blocks) > 1 or block is graph.startblock]
-
-    visited = 0
-
-    while todo:
-        block, cache, inputlink = todo.pop()
-        visited += 1
-        if cache is None:
-            cache = {}
-
-        if block.operations:
-            changed_block = _storesink_block(block, cache, inputlink)
-            added_some_same_as = changed_block or added_some_same_as
-        for link in block.exits:
-            if len(entrymap[link.target]) == 1:
-                new_cache = _translate_cache(cache, link)
-                todo.append((link.target, new_cache, link))
-
-    assert visited == len(entrymap)
-    if added_some_same_as:
-        removenoops.remove_same_as(graph)
-        simplify.transform_dead_op_vars(graph)
-
-def _translate_cache(cache, link):
-    if link.target.operations == (): # exit or except block:
-        return {}
-    block = link.target
-    local_versions = {var1: var2 for var1, var2 in zip(link.args, block.inputargs)}
-    def _translate_arg(arg):
-        if isinstance(arg, Variable):
-            res = local_versions.get(arg, None)
-            if res is None:
-                res = Variable(arg)
-                res.concretetype = arg.concretetype
-                link.args.append(arg)
-                block.inputargs.append(res)
-                local_versions[arg] = res
-            return res
-        else:
-            return arg
-    new_cache = {}
-    for (var, field), res in cache.iteritems():
-        if var in local_versions or not isinstance(var, Variable):
-            new_cache[_translate_arg(var), field] = _translate_arg(res)
-    return new_cache
-
-def _storesink_block(block, cache, inputlink):
-    def clear_cache_for(cache, concretetype, fieldname):
-        for k in cache.keys():
-            if k[0].concretetype == concretetype and k[1] == fieldname:
-                del cache[k]
-
-    added_some_same_as = False
-    for op in block.operations:
-        if op.opname == 'getfield':
-            tup = (op.args[0], op.args[1].value)
-            res = cache.get(tup, None)
-            if res is not None:
-                op.opname = 'same_as'
-                op.args = [res]
-                added_some_same_as = True
-            else:
-                cache[tup] = op.result
-        elif op.opname in ('setarrayitem', 'setinteriorfield', "malloc", "malloc_varsize"):
-            pass
-        elif op.opname == 'setfield':
-            target = op.args[0]
-            field = op.args[1].value
-            clear_cache_for(cache, target.concretetype, field)
-            cache[target, field] = op.args[2]
-        elif has_side_effects(op):
-            cache.clear()
-    return added_some_same_as
diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py
deleted file mode 100644
--- a/rpython/translator/backendopt/test/test_storesink.py
+++ /dev/null
@@ -1,167 +0,0 @@
-
-import py
-from rpython.translator.translator import TranslationContext, graphof
-from rpython.translator.backendopt.storesink import storesink_graph
-from rpython.translator.backendopt import removenoops
-from rpython.flowspace.model import checkgraph
-from rpython.conftest import option
-
-class TestStoreSink(object):
-    def translate(self, func, argtypes):
-        t = TranslationContext()
-        t.buildannotator().build_types(func, argtypes)
-        t.buildrtyper().specialize()
-        return t
-
-    def check(self, f, argtypes, no_getfields=0):
-        t = self.translate(f, argtypes)
-        getfields = 0
-        graph = graphof(t, f)
-        removenoops.remove_same_as(graph)
-        checkgraph(graph)
-        storesink_graph(graph)
-        checkgraph(graph)
-        if option.view:
-            t.view()
-        for block in graph.iterblocks():
-            for op in block.operations:
-                if op.opname == 'getfield':
-                    getfields += 1
-        if no_getfields != getfields:
-            py.test.fail("Expected %d, got %d getfields" %
-                         (no_getfields, getfields))
-
-    def test_infrastructure(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            a.x = i
-            return a.x
-
-        self.check(f, [int], 0)
-
-    def test_simple(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            a.x = i
-            return a.x + a.x
-
-        self.check(f, [int], 0)
-
-    def test_irrelevant_setfield(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            a.x = i
-            one = a.x
-            a.y = 3
-            two = a.x
-            return one + two
-
-        self.check(f, [int], 0)
-
-    def test_relevant_setfield(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            b = A()
-            a.x = i
-            b.x = i + 1
-            one = a.x
-            b.x = i
-            two = a.x
-            return one + two
-
-        self.check(f, [int], 2)
-
-    def test_different_concretetype(self):
-        class A(object):
-            pass
-
-        class B(object):
-            pass
-
-        def f(i):
-            a = A()
-            b = B()
-            a.x = i
-            one = a.x
-            b.x = i + 1
-            two = a.x
-            return one + two
-
-        self.check(f, [int], 0)
-
-    def test_subclass(self):
-        class A(object):
-            pass
-
-        class B(A):
-            pass
-
-        def f(i):
-            a = A()
-            b = B()
-            a.x = i
-            one = a.x
-            b.x = i + 1
-            two = a.x
-            return one + two
-
-        self.check(f, [int], 1)
-
-    def test_bug_1(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            a.cond = i > 0
-            n = a.cond
-            if a.cond:
-                return True
-            return n
-
-        self.check(f, [int], 0)
-
-
-    def test_cfg_splits(self):
-        class A(object):
-            pass
-
-        def f(i):
-            a = A()
-            j = i
-            for i in range(i):
-                a.x = i
-                if i:
-                    j = a.x + a.x
-                else:
-                    j = a.x * 5
-            return j
-
-        self.check(f, [int], 0)
-
-    def test_malloc_does_not_invalidate(self):
-        class A(object):
-            pass
-        class B(object):
-            pass
-
-        def f(i):
-            a = A()
-            a.x = i
-            b = B()
-            return a.x
-
-        self.check(f, [int], 0)
-

From pypy.commits at gmail.com  Mon Nov 21 03:17:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 00:17:58 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2425: don't pass interp-level
 Nones as arguments to call_method()
Message-ID: <5832adb6.46052e0a.7ebf6.3bb4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88505:1bff6ff8dfeb
Date: 2016-11-21 09:04 +0100
http://bitbucket.org/pypy/pypy/changeset/1bff6ff8dfeb/

Log:	Issue #2425: don't pass interp-level Nones as arguments to
	call_method()

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -8,6 +8,7 @@
 from rpython.rlib.rstring import StringBuilder
 from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors
 from pypy.module.posix.interp_posix import dispatch_filename
+from pypy.module.sys.interp_encoding import getdefaultencoding
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
     interp_attrproperty, make_weakref_descr, interp_attrproperty_w)
@@ -309,11 +310,14 @@
             data = space.getarg_w('s*', w_data).as_str()
         else:
             if space.isinstance_w(w_data, space.w_unicode):
-                w_errors = w_encoding = None
                 if self.encoding:
                     w_encoding = space.wrap(self.encoding)
+                else:
+                    w_encoding = getdefaultencoding(space)
                 if self.errors:
                     w_errors = space.wrap(self.errors)
+                else:
+                    w_errors = space.wrap("strict")
                 w_data = space.call_method(w_data, "encode",
                                            w_encoding, w_errors)
             data = space.charbuf_w(w_data)

From pypy.commits at gmail.com  Mon Nov 21 03:50:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 00:50:32 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2425 bis: call encode() when we
 don't have the lock acquired
Message-ID: <5832b558.8dd4190a.b09d6.3ee5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88506:de25d7d491be
Date: 2016-11-21 09:44 +0100
http://bitbucket.org/pypy/pypy/changeset/de25d7d491be/

Log:	Issue #2425 bis: call encode() when we don't have the lock acquired

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -303,27 +303,7 @@
             size = space.r_longlong_w(w_size)
         stream.truncate(size)
 
-    def direct_write(self, w_data):
-        space = self.space
-        self.check_writable()
-        if self.binary:
-            data = space.getarg_w('s*', w_data).as_str()
-        else:
-            if space.isinstance_w(w_data, space.w_unicode):
-                if self.encoding:
-                    w_encoding = space.wrap(self.encoding)
-                else:
-                    w_encoding = getdefaultencoding(space)
-                if self.errors:
-                    w_errors = space.wrap(self.errors)
-                else:
-                    w_errors = space.wrap("strict")
-                w_data = space.call_method(w_data, "encode",
-                                           w_encoding, w_errors)
-            data = space.charbuf_w(w_data)
-        self.do_direct_write(data)
-
-    def do_direct_write(self, data):
+    def direct_write_str(self, data):
         self.softspace = 0
         self.getstream().write(data)
 
@@ -353,7 +333,7 @@
     _exposed_method_names = []
 
     def _decl(class_scope, name, docstring,
-              wrapresult="space.wrap(result)"):
+              wrapresult="space.wrap(result)", exposed=True):
         # hack hack to build a wrapper around the direct_xxx methods.
         # The wrapper adds lock/unlock calls and a space.wrap() on
         # the result, conversion of stream errors to OperationErrors,
@@ -393,7 +373,8 @@
         exec str(src) in globals(), class_scope
         if unwrap_spec is not None:
             class_scope['file_' + name].unwrap_spec = unwrap_spec
-        class_scope['_exposed_method_names'].append(name)
+        if exposed:
+            class_scope['_exposed_method_names'].append(name)
 
 
     _decl(locals(), "__init__", """Opens a file.""")
@@ -470,11 +451,8 @@
 
 Size defaults to the current file position, as returned by tell().""")
 
-    _decl(locals(), "write",
-        """write(str) -> None.  Write string str to file.
-
-Note that due to buffering, flush() or close() may be needed before
-the file on disk reflects the data written.""")
+    _decl(locals(), "write_str", "Interp-level only, see file_write()",
+          exposed=False)
 
     _decl(locals(), "__iter__",
         """Iterating over files, as in 'for line in f:', returns each line of
@@ -505,6 +483,32 @@
         else:
             return space.str_w(space.repr(w_name))
 
+    def file_write(self, w_data):
+        """write(str) -> None.  Write string str to file.
+
+Note that due to buffering, flush() or close() may be needed before
+the file on disk reflects the data written."""
+        space = self.space
+        self.check_writable()
+        if self.binary:
+            data = space.getarg_w('s*', w_data).as_str()
+        else:
+            if space.isinstance_w(w_data, space.w_unicode):
+                # note: "encode" is called before we acquire the lock
+                # for this file, which is done in file_write_str()
+                if self.encoding:
+                    w_encoding = space.wrap(self.encoding)
+                else:
+                    w_encoding = getdefaultencoding(space)
+                if self.errors:
+                    w_errors = space.wrap(self.errors)
+                else:
+                    w_errors = space.wrap("strict")
+                w_data = space.call_method(w_data, "encode",
+                                           w_encoding, w_errors)
+            data = space.charbuf_w(w_data)
+        self.file_write_str(data)
+
     def file_writelines(self, w_lines):
         """writelines(sequence_of_strings) -> None.  Write the strings to the file.
 
@@ -616,6 +620,7 @@
                               cls=W_File,
                               doc="Support for 'print'."),
     __repr__ = interp2app(W_File.file__repr__),
+    write      = interp2app(W_File.file_write),
     writelines = interp2app(W_File.file_writelines),
     __exit__ = interp2app(W_File.file__exit__),
     __weakref__ = make_weakref_descr(W_File),
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -120,7 +120,7 @@
 
 class DirectStreamWriter(StreamReaderWriter):
     def write(self, data):
-        self.file.do_direct_write(data)
+        self.file.direct_write_str(data)
 
 class DirectStreamReader(StreamReaderWriter):
     def read(self, n):

From pypy.commits at gmail.com  Mon Nov 21 03:50:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 00:50:34 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2425: actually,
 don't call the "encode" method generically.
Message-ID: <5832b55a.d5092e0a.5d9b0.3773@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88507:8c314acb8dd4
Date: 2016-11-21 09:48 +0100
http://bitbucket.org/pypy/pypy/changeset/8c314acb8dd4/

Log:	Issue #2425: actually, don't call the "encode" method generically.
	CPython doesn't do that when passed subclasses of unicode. That's
	why I have a hard time coming up with small tests that test anything
	on CPython.

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -8,7 +8,6 @@
 from rpython.rlib.rstring import StringBuilder
 from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors
 from pypy.module.posix.interp_posix import dispatch_filename
-from pypy.module.sys.interp_encoding import getdefaultencoding
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
     interp_attrproperty, make_weakref_descr, interp_attrproperty_w)
@@ -495,17 +494,12 @@
         else:
             if space.isinstance_w(w_data, space.w_unicode):
                 # note: "encode" is called before we acquire the lock
-                # for this file, which is done in file_write_str()
-                if self.encoding:
-                    w_encoding = space.wrap(self.encoding)
-                else:
-                    w_encoding = getdefaultencoding(space)
-                if self.errors:
-                    w_errors = space.wrap(self.errors)
-                else:
-                    w_errors = space.wrap("strict")
-                w_data = space.call_method(w_data, "encode",
-                                           w_encoding, w_errors)
+                # for this file, which is done in file_write_str().
+                # Direct access to unicodeobject because we don't want
+                # to call user-defined "encode" methods here.
+                from pypy.objspace.std.unicodeobject import encode_object
+                w_data = encode_object(space, w_data, self.encoding,
+                                       self.errors)
             data = space.charbuf_w(w_data)
         self.file_write_str(data)
 

From pypy.commits at gmail.com  Mon Nov 21 05:07:05 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:07:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: build openssl cffi backend now as
 _pypy_openssl (to not clash with cryptography)
Message-ID: <5832c749.9f15190a.2e5ad.3d85@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88508:359fdf344b44
Date: 2016-11-21 11:06 +0100
http://bitbucket.org/pypy/pypy/changeset/359fdf344b44/

Log:	build openssl cffi backend now as _pypy_openssl (to not clash with
	cryptography) several changes while discussing the PR back to
	cryptography. most of the functions are now contained in
	cryptography.

diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -1738,6 +1738,10 @@
         return ret
 
     def test_handshake(self):
+        # NOTE: this test has been modified, CPython in newer versions
+        # removed the ability to get the shared ciphers of the session, but
+        # they always return the cipher of the ssl context. This test is fully
+        # removed in later versions
         with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
             sock.connect((REMOTE_HOST, 443))
@@ -1750,13 +1754,13 @@
             sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
@@ -3282,18 +3286,31 @@
             self.assertIn("TypeError", stderr.getvalue())
 
         def test_shared_ciphers(self):
+            # NOTE: This test case has been copied from master (~3.6)
+            # reason is the method SSL_get_ciphers (before that it used
+            # get_ciphers of the session which is not public)
             server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             server_context.load_cert_chain(SIGNED_CERTFILE)
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            client_context.set_ciphers("RC4")
-            server_context.set_ciphers("AES:RC4")
+            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
+                client_context.set_ciphers("AES128:AES256")
+                server_context.set_ciphers("AES256")
+                alg1 = "AES256"
+                alg2 = "AES-256"
+            else:
+                client_context.set_ciphers("AES:3DES")
+                server_context.set_ciphers("3DES")
+                alg1 = "3DES"
+                alg2 = "DES-CBC3"
+
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                self.assertIn("RC4", name.split("-"))
+                if not alg1 in name.split("-") and alg2 not in name:
+                    self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -1,8 +1,16 @@
 # PyPy's SSL module
 
-Most of the CFFI code is copied from cryptography
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
 
 # Tests?
 
 Currently this module is tested using CPython's standard library test suite.
 
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -73,7 +73,6 @@
         "rand",
         "rsa",
         "ssl",
-        "tls1",
         "x509",
         "x509name",
         "x509v3",
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/bio.py
@@ -82,11 +82,12 @@
 int BIO_write(BIO *, const void *, int);
 int BIO_puts(BIO *, const char *);
 int BIO_method_type(const BIO *);
-
-int * Cryptography_bio_references(const BIO *);
 """
 
 MACROS = """
+/* Added in 1.1.0 */
+int BIO_up_ref(BIO *);
+
 /* These added const to BIO_METHOD in 1.1.0 */
 BIO *BIO_new(BIO_METHOD *);
 BIO_METHOD *BIO_s_mem(void);
@@ -134,12 +135,13 @@
 long BIO_set_nbio(BIO *, long);
 void BIO_set_retry_read(BIO *);
 void BIO_clear_retry_flags(BIO *);
-
-#define CRYPTO_LOCK_BIO ...
 """
 
 CUSTOMIZATIONS = """
-int * Cryptography_bio_references(const BIO * b) {
-    return &b->references;
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE4 || defined(LIBRESSL_VERSION_NUMBER)
+int BIO_up_ref(BIO *b) {
+    CRYPTO_add(&b->references, 1, CRYPTO_LOCK_BIO);
+    return 1;
 }
+#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
@@ -57,8 +57,6 @@
 
 /* This was removed in 1.1.0 */
 void CRYPTO_lock(int, int, const char *, int);
-
-void CRYPTO_add(void*,int,int);
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py
@@ -36,8 +36,12 @@
     (OPENSSL_VERSION_NUMBER < 0x1000209fL)
 #define CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 \
     (OPENSSL_VERSION_NUMBER < 0x10100000)
+#define CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE4 \
+    (OPENSSL_VERSION_NUMBER < 0x10100004)
 #define CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE5 \
     (OPENSSL_VERSION_NUMBER < 0x10100005)
+#define CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE6 \
+    (OPENSSL_VERSION_NUMBER < 0x10100006)
 
 #if defined(LIBRESSL_VERSION_NUMBER)
 #define CRYPTOGRAPHY_IS_LIBRESSL 1
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/dh.py
@@ -26,6 +26,7 @@
 int i2d_DHparams(const DH *, unsigned char **);
 int DHparams_print_fp(FILE *, const DH *);
 int DHparams_print(BIO *, const DH *);
+DH *DHparams_dup(DH *);
 
 /* added in 1.1.0 when the DH struct was opaqued */
 void DH_get0_pqg(const DH *, const BIGNUM **, const BIGNUM **,
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ecdh.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ecdh.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ecdh.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ecdh.py
@@ -12,7 +12,7 @@
 
 TYPES = """
 static const int Cryptography_HAS_ECDH;
-static const int Cryptography_HAS_ECDH_SET_CURVE;
+static const int Cryptography_HAS_SET_ECDH_AUTO;
 """
 
 FUNCTIONS = """
@@ -21,7 +21,7 @@
 MACROS = """
 int ECDH_compute_key(void *, size_t, const EC_POINT *, EC_KEY *,
                      void *(*)(const void *, size_t, void *, size_t *));
-int SSL_CTX_set_ecdh_auto(SSL_CTX *ctx, int onoff);
+int SSL_CTX_set_ecdh_auto(SSL_CTX *, int);
 """
 
 CUSTOMIZATIONS = """
@@ -35,9 +35,11 @@
 #else
 static const long Cryptography_HAS_ECDH = 1;
 #endif
-#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 || defined(LIBRESSL_VERSION_NUMBER)
-static const long Cryptography_HAS_ECDH_SET_CURVE = 0;
+
+#ifndef SSL_CTX_set_ecdh_auto
+static const long Cryptography_HAS_SET_ECDH_AUTO = 0;
+int (*SSL_CTX_set_ecdh_auto)(SSL_CTX *, int) = NULL;
 #else
-static const long Cryptography_HAS_ECDH_SET_CURVE = 1;
+static const long Cryptography_HAS_SET_ECDH_AUTO = 1;
 #endif
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/err.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/err.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/err.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/err.py
@@ -81,6 +81,7 @@
 static const int ASN1_R_NO_CONTENT_TYPE;
 static const int ASN1_R_NO_MULTIPART_BODY_FAILURE;
 static const int ASN1_R_NO_MULTIPART_BOUNDARY;
+static const int ASN1_R_HEADER_TOO_LONG;
 
 static const int DH_F_COMPUTE_KEY;
 
@@ -224,6 +225,8 @@
 static const int SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE;
 static const int SSL_AD_BAD_CERTIFICATE_HASH_VALUE;
 static const int SSL_AD_UNKNOWN_PSK_IDENTITY;
+
+static const int X509_R_CERT_ALREADY_IN_HASH_TABLE;
 """
 
 FUNCTIONS = """
@@ -280,24 +283,4 @@
 static const long Cryptography_HAS_RSA_R_PKCS_DECODING_ERROR = 0;
 static const long RSA_R_PKCS_DECODING_ERROR = 0;
 #endif
-
-#ifndef SSL_AD_UNSUPPORTED_EXTENSION
-    static const int SSL_AD_UNSUPPORTED_EXTENSION = -1;
-#endif
-#ifndef SSL_AD_CERTIFICATE_UNOBTAINABLE
-    static const int SSL_AD_CERTIFICATE_UNOBTAINABLE = -1;
-#endif
-#ifndef SSL_AD_UNRECOGNIZED_NAME
-    static const int SSL_AD_UNRECOGNIZED_NAME = -1;
-#endif
-#ifndef SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE
-    static const int SSL_AD_BAD_CERTIFICATE_STATUS_RESPONSE = -1;
-#endif
-#ifndef SSL_AD_BAD_CERTIFICATE_HASH_VALUE
-    static const int SSL_AD_BAD_CERTIFICATE_HASH_VALUE = -1;
-#endif
-#ifndef SSL_AD_UNKNOWN_PSK_IDENTITY
-    static const int SSL_AD_UNKNOWN_PSK_IDENTITY = -1;
-#endif
-
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/nid.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/nid.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/nid.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/nid.py
@@ -215,14 +215,15 @@
 static const int NID_pseudonym;
 static const int NID_domainComponent;
 static const int NID_pkcs9_emailAddress;
+
+static const int NID_ad_OCSP;
+static const int NID_ad_ca_issuers;
 """
 
 FUNCTIONS = """
 """
 
 MACROS = """
-static const int NID_ad_OCSP;
-static const int NID_ad_ca_issuers;
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/rand.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/rand.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/rand.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/rand.py
@@ -20,7 +20,6 @@
 int RAND_load_file(const char *, long);
 int RAND_write_file(const char *);
 int RAND_bytes(unsigned char *, int);
-int RAND_pseudo_bytes(unsigned char *buf, int num);
 """
 
 MACROS = """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -26,9 +26,7 @@
 static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE;
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS;
 static const long Cryptography_HAS_NPN_NEGOTIATED;
-static const long Cryptography_NO_TLSEXT;
-
-static const long Cryptography_OPENSSL_NPN_NEGOTIATED;
+static const long Cryptography_OPENSSL_NO_TLSEXT;
 
 /* Internally invented symbol to tell us if SNI is supported */
 static const long Cryptography_HAS_TLSEXT_HOSTNAME;
@@ -134,6 +132,8 @@
 static const long TLS_ST_BEFORE;
 static const long TLS_ST_OK;
 
+static const long OPENSSL_NPN_NEGOTIATED;
+
 typedef ... SSL_METHOD;
 typedef ... SSL_CTX;
 
@@ -196,7 +196,6 @@
 int SSL_renegotiate_pending(SSL *);
 const char *SSL_get_cipher_list(const SSL *, int);
 Cryptography_STACK_OF_SSL_CIPHER *SSL_get_ciphers(const SSL *);
-Cryptography_STACK_OF_SSL_CIPHER * Cryptography_get_ssl_session_ciphers(const SSL_SESSION *);
 
 /*  context */
 void SSL_CTX_free(SSL_CTX *);
@@ -691,17 +690,9 @@
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1;
 
 #ifdef OPENSSL_NO_TLSEXT
-static const long Cryptography_NO_TLSEXT = 1;
+static const long Cryptography_OPENSSL_NO_TLSEXT = 1;
 #else
-static const long Cryptography_NO_TLSEXT = 0;
-#endif
-
-#ifdef OPENSSL_NPN_NEGOTIATED
-static const long Cryptography_OPENSSL_NPN_NEGOTIATED = OPENSSL_NPN_NEGOTIATED;
-static const long Cryptography_HAS_NPN_NEGOTIATED = 1;
-#else
-static const long Cryptography_OPENSSL_NPN_NEGOTIATED = 0;
-static const long Cryptography_HAS_NPN_NEGOTIATED = 0;
+static const long Cryptography_OPENSSL_NO_TLSEXT = 0;
 #endif
 
 /* in OpenSSL 1.1.0 the SSL_ST values were renamed to TLS_ST and several were
@@ -723,7 +714,12 @@
 static const long TLS_ST_OK = 0;
 #endif
 
-Cryptography_STACK_OF_SSL_CIPHER * Cryptography_get_ssl_session_ciphers(const SSL_SESSION *s) {
-    return s->ciphers;
-}
+/* This define is available in 1.0.1+ so we can remove this when we drop
+   support for 1.0.0 */
+#ifdef OPENSSL_NPN_NEGOTIATED
+static const long Cryptography_HAS_NPN_NEGOTIATED = 1;
+#else
+static const long OPENSSL_NPN_NEGOTIATED = -1;
+static const long Cryptography_HAS_NPN_NEGOTIATED = 0;
+#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/tls1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/tls1.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/tls1.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/tls1.py
@@ -12,7 +12,8 @@
 """
 
 FUNCTIONS = """
-long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX *, int(*)(SSL*,int*,void*));
+long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX *,
+                                            int(*)(SSL*,int*,void*));
 long Cryptography_SSL_CTX_set_tlsext_servername_arg(SSL_CTX *, void * arg);
 """
 
@@ -20,7 +21,8 @@
 """
 
 CUSTOMIZATIONS = """
-long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX * ctx, int(*cb)(SSL*,int*,void*)) {
+long Cryptography_SSL_CTX_set_tlsext_servername_callback(SSL_CTX * ctx,
+                                              int(*cb)(SSL*,int*,void*)) {
     return SSL_CTX_set_tlsext_servername_callback(ctx, cb);
 }
 long Cryptography_SSL_CTX_set_tlsext_servername_arg(SSL_CTX * ctx, void *arg) {
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509.py
@@ -17,15 +17,12 @@
 typedef STACK_OF(X509) Cryptography_STACK_OF_X509;
 typedef STACK_OF(X509_CRL) Cryptography_STACK_OF_X509_CRL;
 typedef STACK_OF(X509_REVOKED) Cryptography_STACK_OF_X509_REVOKED;
-typedef STACK_OF(ACCESS_DESCRIPTION) Cryptography_AUTHORITY_INFO_ACCESS;
-typedef STACK_OF(X509_OBJECT) Cryptography_STACK_OF_X509_OBJECT;
 """
 
 TYPES = """
 typedef ... Cryptography_STACK_OF_X509;
 typedef ... Cryptography_STACK_OF_X509_CRL;
 typedef ... Cryptography_STACK_OF_X509_REVOKED;
-typedef ... Cryptography_AUTHORITY_INFO_ACCESS;
 
 typedef struct {
     ASN1_OBJECT *algorithm;
@@ -339,11 +336,6 @@
 int i2d_re_X509_CRL_tbs(X509_CRL *, unsigned char **);
 void X509_REQ_get0_signature(const X509_REQ *, const ASN1_BIT_STRING **,
                              const X509_ALGOR **);
-
-void AUTHORITY_INFO_ACCESS_free(Cryptography_AUTHORITY_INFO_ACCESS*);
-
-#define X509_R_CERT_ALREADY_IN_HASH_TABLE ...
-#define ASN1_R_HEADER_TOO_LONG ...
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
@@ -23,17 +23,12 @@
 static const long Cryptography_HAS_102_VERIFICATION_PARAMS;
 static const long Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST;
 static const long Cryptography_HAS_X509_V_FLAG_PARTIAL_CHAIN;
-static const long Cryptography_X509_LU_X509;
-static const long Cryptography_X509_LU_CLR;
 
 typedef ... Cryptography_STACK_OF_ASN1_OBJECT;
 typedef ... Cryptography_STACK_OF_X509_OBJECT;
 
 typedef ... X509_OBJECT;
-typedef struct {
-    Cryptography_STACK_OF_X509_OBJECT * objs;
-    ...;
-} X509_STORE;
+typedef ... X509_STORE;
 typedef ... X509_VERIFY_PARAM;
 typedef ... X509_STORE_CTX;
 
@@ -128,6 +123,8 @@
 static const long X509_V_FLAG_SUITEB_128_LOS;
 static const long X509_V_FLAG_PARTIAL_CHAIN;
 
+static const long X509_LU_X509;
+static const long X509_LU_CRL;
 """
 
 FUNCTIONS = """
@@ -142,7 +139,6 @@
 int X509_STORE_set_default_paths(X509_STORE *);
 int X509_STORE_set_flags(X509_STORE *, unsigned long);
 void X509_STORE_free(X509_STORE *);
-X509_VERIFY_PARAM *_X509_STORE_get0_param(X509_STORE *);
 
 /* X509_STORE_CTX */
 X509_STORE_CTX *X509_STORE_CTX_new(void);
@@ -182,8 +178,6 @@
 void X509_VERIFY_PARAM_set_depth(X509_VERIFY_PARAM *, int);
 int X509_VERIFY_PARAM_get_depth(const X509_VERIFY_PARAM *);
 void X509_VERIFY_PARAM_free(X509_VERIFY_PARAM *);
-int Cryptography_X509_OBJECT_get_type(const X509_OBJECT *);
-X509 * Cryptography_X509_OBJECT_data_x509(X509_OBJECT*);
 """
 
 MACROS = """
@@ -205,10 +199,12 @@
                               size_t);
 int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *, const char *);
 
-/* STACK_OF(X509_OBJECT) */
 int sk_X509_OBJECT_num(Cryptography_STACK_OF_X509_OBJECT *);
 X509_OBJECT *sk_X509_OBJECT_value(Cryptography_STACK_OF_X509_OBJECT *, int);
-
+X509_VERIFY_PARAM * X509_STORE_get0_param(X509_STORE *);
+Cryptography_STACK_OF_X509_OBJECT *X509_STORE_get0_objects(X509_STORE *);
+X509 *X509_OBJECT_get0_X509(X509_OBJECT *);
+int X509_OBJECT_get_type(const X509_OBJECT *);
 """
 
 CUSTOMIZATIONS = """
@@ -268,18 +264,21 @@
 static const long X509_V_FLAG_TRUSTED_FIRST = 0;
 #endif
 
-X509_VERIFY_PARAM *_X509_STORE_get0_param(X509_STORE *store) {
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE6 || defined(LIBRESSL_VERSION_NUMBER)
+Cryptography_STACK_OF_X509_OBJECT *X509_STORE_get0_objects(X509_STORE *ctx) {
+    return ctx->objs;
+}
+X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *store) {
     return store->param;
 }
-
-int Cryptography_X509_OBJECT_get_type(const X509_OBJECT * x) {
+int X509_OBJECT_get_type(const X509_OBJECT *x) {
     return x->type;
 }
+#endif
 
-X509 * Cryptography_X509_OBJECT_data_x509(X509_OBJECT * x) {
+#if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110PRE5 || defined(LIBRESSL_VERSION_NUMBER)
+X509 *X509_OBJECT_get0_X509(X509_OBJECT *x) {
     return x->data.x509;
 }
-
-static const long Cryptography_X509_LU_X509 = X509_LU_X509;
-static const long Cryptography_X509_LU_CLR = X509_LU_CRL;
+#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509v3.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509v3.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509v3.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509v3.py
@@ -171,25 +171,23 @@
     Cryptography_STACK_OF_POLICYQUALINFO *qualifiers;
 } POLICYINFO;
 
-typedef void (*sk_GENERAL_NAME_freefunc)(struct GENERAL_NAME_st *);
+typedef void (*sk_GENERAL_NAME_freefunc)(GENERAL_NAME *);
 """
 
 
 FUNCTIONS = """
 int X509V3_EXT_add_alias(int, int);
 void X509V3_set_ctx(X509V3_CTX *, X509 *, X509 *, X509_REQ *, X509_CRL *, int);
-GENERAL_NAME *GENERAL_NAME_new(void);
 int GENERAL_NAME_print(BIO *, GENERAL_NAME *);
 GENERAL_NAMES *GENERAL_NAMES_new(void);
 void GENERAL_NAMES_free(GENERAL_NAMES *);
 void *X509V3_EXT_d2i(X509_EXTENSION *);
-/* X509 is private, there is no way to access the field crldp other than
-   adding it to the typedef or expose a function like this: */
-Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 *);
 int X509_check_ca(X509 *);
 """
 
 MACROS = """
+/* X509 became a const arg in 1.1.0 */
+void *X509_get_ext_d2i(X509 *, int, int *, int *);
 /* The last two char * args became const char * in 1.1.0 */
 X509_EXTENSION *X509V3_EXT_nconf(CONF *, X509V3_CTX *, char *, char *);
 /* This is a macro defined by a call to DECLARE_ASN1_FUNCTIONS in the
@@ -219,9 +217,8 @@
 int sk_GENERAL_NAME_num(struct stack_st_GENERAL_NAME *);
 int sk_GENERAL_NAME_push(struct stack_st_GENERAL_NAME *, GENERAL_NAME *);
 GENERAL_NAME *sk_GENERAL_NAME_value(struct stack_st_GENERAL_NAME *, int);
-
-void sk_GENERAL_NAME_pop_free(struct stack_st_GENERAL_NAME*, sk_GENERAL_NAME_freefunc);
-void GENERAL_NAME_free(struct GENERAL_NAME_st*);
+void sk_GENERAL_NAME_pop_free(struct stack_st_GENERAL_NAME *,
+                              sk_GENERAL_NAME_freefunc);
 
 Cryptography_STACK_OF_ACCESS_DESCRIPTION *sk_ACCESS_DESCRIPTION_new_null(void);
 int sk_ACCESS_DESCRIPTION_num(Cryptography_STACK_OF_ACCESS_DESCRIPTION *);
@@ -299,17 +296,9 @@
 DIST_POINT_NAME *DIST_POINT_NAME_new(void);
 void DIST_POINT_NAME_free(DIST_POINT_NAME *);
 
-void * X509_get_ext_d2i(const X509 *, int, int *, int *);
+GENERAL_NAME *GENERAL_NAME_new(void);
+void GENERAL_NAME_free(GENERAL_NAME *);
 """
 
 CUSTOMIZATIONS = """
-#if OPENSSL_VERSION_NUMBER >= 0x10001000L
-Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 * x) {
-    return x->crldp;
-}
-#else
-Cryptography_STACK_OF_DIST_POINT * Cryptography_X509_get_crldp(const X509 * x) {
-    return NULL;
-}
-#endif
 """
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -3,8 +3,8 @@
 import _thread
 import socket
 import weakref
-from _openssl import ffi
-from _openssl import lib
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
 from _cffi_ssl._stdssl.certificate import (_test_decode_cert,
     _decode_certificate, _certificate_to_der)
 from _cffi_ssl._stdssl.utility import (_str_with_len, _bytes_with_len,
@@ -37,7 +37,7 @@
 HAS_ECDH = bool(lib.Cryptography_HAS_ECDH)
 HAS_SNI = bool(lib.Cryptography_HAS_TLSEXT_HOSTNAME)
 HAS_ALPN = bool(lib.Cryptography_HAS_ALPN)
-HAS_NPN = lib.Cryptography_HAS_NPN_NEGOTIATED
+HAS_NPN = bool(lib.OPENSSL_NPN_NEGOTIATED)
 HAS_TLS_UNIQUE = True
 
 CLIENT = 0
@@ -225,10 +225,8 @@
             # BIOs are reference counted and SSL_set_bio borrows our reference.
             # To prevent a double free in memory_bio_dealloc() we need to take an
             # extra reference here.
-            irefaddr = lib.Cryptography_bio_references(inbio.bio);
-            orefaddr = lib.Cryptography_bio_references(outbio.bio);
-            lib.CRYPTO_add(irefaddr, 1, lib.CRYPTO_LOCK_BIO)
-            lib.CRYPTO_add(orefaddr, 1, lib.CRYPTO_LOCK_BIO)
+            lib.BIO_up_ref(inbio.bio);
+            lib.BIO_up_ref(outbio.bio);
             lib.SSL_set_bio(self.ssl, inbio.bio, outbio.bio)
 
         mode = lib.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER
@@ -271,8 +269,6 @@
         self._owner = None
         self.server_hostname = None
         self.socket = None
-        #self.alpn_protocols = ffi.NULL
-        #self.npn_protocols = ffi.NULL
 
     @property
     def owner(self):
@@ -515,10 +511,7 @@
             return _str_with_len(out[0], outlen[0]);
 
     def shared_ciphers(self):
-        sess = lib.SSL_get_session(self.ssl)
-        if sess == ffi.NULL:
-            return None
-        ciphers = lib.Cryptography_get_ssl_session_ciphers(sess)
+        ciphers = lib.SSL_get_ciphers(self.ssl)
         if ciphers == ffi.NULL:
             return None
         res = []
@@ -732,7 +725,8 @@
 
 class _SSLContext(object):
     __slots__ = ('ctx', '_check_hostname', 'servername_callback',
-                 'alpn_protocols', 'npn_protocols', 'set_hostname',
+                 'alpn_protocols', '_alpn_protocols_handle',
+                 'npn_protocols', 'set_hostname',
                  '_set_hostname_handle', '_npn_protocols_handle')
 
     def __new__(cls, protocol):
@@ -758,7 +752,6 @@
             raise ssl_error("failed to allocate SSL context")
 
         self._check_hostname = False
-        # TODO self.register_finalizer(space)
 
         # Defaults
         lib.SSL_CTX_set_verify(self.ctx, lib.SSL_VERIFY_NONE, ffi.NULL)
@@ -775,7 +768,7 @@
             # OpenSSL 1.0.2+), or use prime256v1 by default.
             # This is Apache mod_ssl's initialization
             # policy, so we should be safe.
-            if lib.Cryptography_HAS_ECDH_SET_CURVE:
+            if lib.Cryptography_HAS_SET_ECDH_AUTO:
                 lib.SSL_CTX_set_ecdh_auto(self.ctx, 1)
             else:
                 key = lib.EC_KEY_new_by_curve_name(lib.NID_X9_62_prime256v1)
@@ -834,7 +827,7 @@
     @property
     def verify_flags(self):
         store = lib.SSL_CTX_get_cert_store(self.ctx)
-        param = lib._X509_STORE_get0_param(store)
+        param = lib.X509_STORE_get0_param(store)
         flags = lib.X509_VERIFY_PARAM_get_flags(param)
         return int(flags)
 
@@ -842,16 +835,16 @@
     def verify_flags(self, value):
         new_flags = int(value)
         store = lib.SSL_CTX_get_cert_store(self.ctx);
-        param = lib._X509_STORE_get0_param(store)
+        param = lib.X509_STORE_get0_param(store)
         flags = lib.X509_VERIFY_PARAM_get_flags(param);
         clear = flags & ~new_flags;
         set = ~flags & new_flags;
         if clear:
-            param = lib._X509_STORE_get0_param(store)
+            param = lib.X509_STORE_get0_param(store)
             if not lib.X509_VERIFY_PARAM_clear_flags(param, clear):
                 raise ssl_error(None, 0)
         if set:
-            param = lib._X509_STORE_get0_param(store)
+            param = lib.X509_STORE_get0_param(store)
             if not lib.X509_VERIFY_PARAM_set_flags(param, set):
                 raise ssl_error(None, 0)
 
@@ -1036,17 +1029,17 @@
         x509 = 0
         x509_ca = 0
         crl = 0
-        objs = store.objs
+        objs = lib.X509_STORE_get0_objects(store)
         count = lib.sk_X509_OBJECT_num(objs)
         for i in range(count):
             obj = lib.sk_X509_OBJECT_value(objs, i)
-            _type = lib.Cryptography_X509_OBJECT_get_type(obj)
-            if _type == lib.Cryptography_X509_LU_X509:
+            _type = lib.X509_OBJECT_get_type(obj)
+            if _type == lib.X509_LU_X509:
                 x509 += 1
-                cert = lib.Cryptography_X509_OBJECT_data_x509(obj)
+                cert = lib.X509_OBJECT_get0_X509(obj)
                 if lib.X509_check_ca(cert):
                     x509_ca += 1
-            elif _type == lib.Cryptography_X509_LU_CRL:
+            elif _type == lib.X509_LU_CRL:
                 crl += 1
             else:
                 # Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY.
@@ -1056,12 +1049,14 @@
         return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
 
 
-#    REVIEW, how to do that properly
-#    def _finalize_(self):
-#        ctx = self.ctx
-#        if ctx:
-#            self.ctx = lltype.nullptr(SSL_CTX.TO)
-#            libssl_SSL_CTX_free(ctx)
+    def __del__(self):
+        # REVIEW, is this done properly? In RPython the a finalizer
+        # function is added
+        ctx = self.ctx
+        if ctx:
+            self.ctx = None
+            lib.SSL_CTX_free(ctx)
+
 
     def session_stats(self):
         stats = {}
@@ -1106,16 +1101,16 @@
         binary_mode = bool(binary_form)
         _list = []
         store = lib.SSL_CTX_get_cert_store(self.ctx)
-        objs = store.objs
+        objs = lib.X509_STORE_get0_objects(store)
         count = lib.sk_X509_OBJECT_num(objs)
         for i in range(count):
             obj = lib.sk_X509_OBJECT_value(objs, i)
-            _type = lib.Cryptography_X509_OBJECT_get_type(obj)
-            if _type != lib.Cryptography_X509_LU_X509:
+            _type = lib.X509_OBJECT_get_type(obj)
+            if _type != lib.X509_LU_X509:
                 # not a x509 cert
                 continue
             # CA for any purpose
-            cert = lib.Cryptography_X509_OBJECT_data_x509(obj)
+            cert = lib.X509_OBJECT_get0_X509(obj)
             if not lib.X509_check_ca(cert):
                 continue
             if binary_mode:
@@ -1138,7 +1133,7 @@
             lib.EC_KEY_free(key)
 
     def set_servername_callback(self, callback):
-        if not HAS_SNI or lib.Cryptography_NO_TLSEXT:
+        if not HAS_SNI or lib.Cryptography_OPENSSL_NO_TLSEXT:
             raise NotImplementedError("The TLS extension servername callback, "
                     "SSL_CTX_set_tlsext_servername_callback, "
                     "is not in the current OpenSSL library.")
@@ -1150,8 +1145,8 @@
             raise TypeError("not a callable object")
         scb = ServernameCallback(callback, self)
         self._set_hostname_handle = ffi.new_handle(scb)
-        lib.Cryptography_SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
-        lib.Cryptography_SSL_CTX_set_tlsext_servername_arg(self.ctx, self._set_hostname_handle)
+        lib.SSL_CTX_set_tlsext_servername_callback(self.ctx, _servername_callback)
+        lib.SSL_CTX_set_tlsext_servername_arg(self.ctx, self._set_hostname_handle)
 
     def _set_alpn_protocols(self, protos):
         if HAS_ALPN:
@@ -1160,7 +1155,7 @@
 
             if lib.SSL_CTX_set_alpn_protos(self.ctx,ffi.cast("unsigned char*", protocols), length):
                 return MemoryError()
-            handle = ffi.new_handle(self)
+            self._alpn_protocols_handle = handle = ffi.new_handle(self)
             lib.SSL_CTX_set_alpn_select_cb(self.ctx, select_alpn_callback, handle)
         else:
             raise NotImplementedError("The ALPN extension requires OpenSSL 1.0.2 or later.")
@@ -1187,7 +1182,7 @@
 
 
 
-if HAS_SNI and not lib.Cryptography_NO_TLSEXT:
+if HAS_SNI and not lib.Cryptography_OPENSSL_NO_TLSEXT:
     @ffi.callback("int(SSL*,int*,void*)")
     def _servername_callback(s, al, arg):
         scb = ffi.from_handle(arg)
@@ -1386,13 +1381,15 @@
         raise ValueError("num must be positive")
     buf = ffi.new("unsigned char[%d]" % count)
     if pseudo:
-        ok = lib.RAND_pseudo_bytes(buf, count)
+        # note by reaperhulk, RAND_pseudo_bytes is deprecated in 3.6 already,
+        # it is totally fine to just call RAND_bytes instead
+        ok = lib.RAND_bytes(buf, count)
         if ok == 1 or ok == 0:
             _bytes = _bytes_with_len(buf, count)
             return (_bytes, ok == 1)
     else:
         ok = lib.RAND_bytes(buf, count)
-        if ok == 1:
+        if ok == 1 or (pseudo and ok == 0):
             return _bytes_with_len(buf, count)
     raise ssl_error(None, errcode=lib.ERR_get_error())
 
@@ -1429,7 +1426,7 @@
                                  ffi.cast("unsigned char*",ctx.alpn_protocols), len(ctx.alpn_protocols),
                                  client_protocols, client_protocols_len)
 
-if lib.Cryptography_HAS_NPN_NEGOTIATED:
+if lib.OPENSSL_NPN_NEGOTIATED:
     @ffi.callback("int(SSL*,unsigned char **,unsigned char *,const unsigned char *,unsigned int,void *)")
     def select_npn_callback(ssl, out, outlen, server_protocols, server_protocols_len, args):
         ctx = ffi.from_handle(args)
@@ -1463,7 +1460,7 @@
         ret = lib.SSL_select_next_proto(out, outlen,
                                         server_protocols, server_protocols_len,
                                         client_protocols, client_protocols_len);
-        if alpn and ret != lib.Cryptography_OPENSSL_NPN_NEGOTIATED:
+        if alpn and ret != lib.OPENSSL_NPN_NEGOTIATED:
             return lib.SSL_TLSEXT_ERR_NOACK
 
         return lib.SSL_TLSEXT_ERR_OK
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -1,8 +1,8 @@
 import warnings
 import base64
 import textwrap
-from _openssl import ffi
-from _openssl import lib
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
 from _cffi_ssl._stdssl.utility import _string_from_asn1, _str_with_len, _bytes_with_len
 from _cffi_ssl._stdssl.error import ssl_error, pyssl_error
 
@@ -30,7 +30,7 @@
     if (info == ffi.NULL):
         return None;
     if lib.sk_ACCESS_DESCRIPTION_num(info) == 0:
-        lib.AUTHORITY_INFO_ACCESS_free(info)
+        lib.sk_ACCESS_DESCRIPTION_free(info)
         return None
 
     lst = []
@@ -44,7 +44,7 @@
         uri = ad.location.d.uniformResourceIdentifier
         ostr = _str_with_len(uri.data, uri.length)
         lst.append(ostr)
-    lib.AUTHORITY_INFO_ACCESS_free(info)
+    lib.sk_ACCESS_DESCRIPTION_free(info)
 
     # convert to tuple or None
     if len(lst) == 0: return None
@@ -244,12 +244,9 @@
 
 
 def _get_crl_dp(certificate):
-    if lib.OPENSSL_VERSION_NUMBER < 0x10001000:
-        dps = lib.X509_get_ext_d2i(certificate, lib.NID_crl_distribution_points, ffi.NULL, ffi.NULL)
-    else:
-        # Calls x509v3_cache_extensions and sets up crldp
+    if lib.OPENSSL_VERSION_NUMBER >= 0x10001000:
         lib.X509_check_ca(certificate)
-        dps = lib.Cryptography_X509_get_crldp(certificate)
+    dps = lib.X509_get_ext_d2i(certificate, lib.NID_crl_distribution_points, ffi.NULL, ffi.NULL)
     if dps is ffi.NULL:
         return None
 
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,7 +1,7 @@
 import sys
 import traceback
-from _openssl import ffi
-from _openssl import lib
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
 
 from _cffi_ssl._stdssl.utility import _string_from_asn1, _str_to_ffi_buffer, _str_from_buf
 from _cffi_ssl._stdssl.errorcodes import _error_codes, _lib_codes
diff --git a/lib_pypy/_cffi_ssl/_stdssl/errorcodes.py b/lib_pypy/_cffi_ssl/_stdssl/errorcodes.py
--- a/lib_pypy/_cffi_ssl/_stdssl/errorcodes.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/errorcodes.py
@@ -1,7 +1,7 @@
 # File generated by tools/make_ssl_data.py
 # Generated on 2016-11-10T17:38:59.402032
 
-from _openssl import ffi, lib 
+from _pypy_openssl import ffi, lib 
 _lib_codes = []
 _lib_codes.append(("PEM", lib.ERR_LIB_PEM))
 _lib_codes.append(("SSL", lib.ERR_LIB_SSL))
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -1,6 +1,6 @@
 import sys
-from _openssl import ffi
-from _openssl import lib
+from _pypy_openssl import ffi
+from _pypy_openssl import lib
 
 def _string_from_asn1(asn1):
     data = lib.ASN1_STRING_data(asn1)
diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
--- a/lib_pypy/_ssl_build.py
+++ b/lib_pypy/_ssl_build.py
@@ -2,7 +2,52 @@
 from _cffi_ssl import _cffi_src
 sys.modules['_cffi_src'] = _cffi_src
 #
-from _cffi_ssl._cffi_src.build_openssl import ffi
+from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding,
+        _get_openssl_libraries, extra_link_args, compiler_type)
+
+ffi = build_ffi_for_binding(
+    module_name="_pypy_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
 
 if __name__ == '__main__':
     ffi.compile()

From pypy.commits at gmail.com  Mon Nov 21 05:24:46 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:24:46 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: merge py3.5
Message-ID: <5832cb6e.83092e0a.8c002.43fb@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88509:363e082d0436
Date: 2016-11-21 11:24 +0100
http://bitbucket.org/pypy/pypy/changeset/363e082d0436/

Log:	merge py3.5

diff too long, truncating to 2000 out of 20922 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,423 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation, BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):

From pypy.commits at gmail.com  Mon Nov 21 05:47:22 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 02:47:22 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: various fixes
Message-ID: <5832d0ba.05052e0a.a5b63.42a5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88510:df80a32ac30f
Date: 2016-11-16 14:11 +0100
http://bitbucket.org/pypy/pypy/changeset/df80a32ac30f/

Log:	various fixes

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -292,7 +292,7 @@
         # obscure hack to stuff the translation options into the translated PyPy
         import pypy.module.sys
         options = make_dict(config)
-        wrapstr = 'space.newtext(%r)' % (options)
+        wrapstr = 'space.wrap(%r)' % (options) # import time
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -19,10 +19,10 @@
 
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
-    w_msg = space.wrap(msg)
+    w_msg = space.newtext(msg)
     w_filename = space.wrap(fn)
-    w_lineno = space.wrap(lineno)
-    w_offset = space.wrap(offset)
+    w_lineno = space.newint(lineno)
+    w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
 
 
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -425,7 +425,6 @@
     @jit.dont_look_inside
     def _reduce_state(self, space):
         from pypy.module._pickle_support import maker # helper fns
-        w = space.wrap
         nt = space.newtuple
 
         if self.get_w_f_trace() is None:
@@ -447,26 +446,26 @@
 
         d = self.getorcreatedebug()
         tup_state = [
-            w(self.f_backref()),
-            w(self.get_builtin()),
-            w(self.pycode),
+            self.f_backref(),
+            self.get_builtin(),
+            self.pycode,
             w_locals_cells_stack,
             w_blockstack,
             w_exc_value, # last_exception
             w_tb,        #
             self.get_w_globals(),
-            w(self.last_instr),
-            w(self.frame_finished_execution),
-            w(f_lineno),
+            space.newint(self.last_instr),
+            space.newbool(self.frame_finished_execution),
+            space.newint(f_lineno),
             space.w_None,           #XXX placeholder for f_locals
 
             #f_restricted requires no additional data!
             space.w_None,
 
-            w(d.instr_lb),
-            w(d.instr_ub),
-            w(d.instr_prev_plus_one),
-            w(self.valuestackdepth),
+            space.newint(d.instr_lb),
+            space.newint(d.instr_ub),
+            space.newint(d.instr_prev_plus_one),
+            space.newint(self.valuestackdepth),
             ]
         return nt(tup_state)
 
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -65,7 +65,7 @@
         # decode input (with the eventual \r from a previous pass)
         if not space.is_w(self.w_decoder, space.w_None):
             w_output = space.call_method(self.w_decoder, "decode",
-                                         w_input, space.newbool(final))
+                                         w_input, space.newbool(bool(final)))
         else:
             w_output = w_input
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -48,7 +48,7 @@
 class UMathModule(MixedModule):
     appleveldefs = {}
     interpleveldefs = {
-        'FLOATING_POINT_SUPPORT': 'space.wrap(1)',
+        'FLOATING_POINT_SUPPORT': 'space.newint(1)',
         'frompyfunc': 'ufuncs.frompyfunc',
         }
     # ufuncs
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -68,7 +68,7 @@
         scalar = multiarray.get("scalar")
 
         ret = space.newtuple([scalar, space.newtuple(
-            [space.wrap(self._get_dtype(space)), space.wrap(self.raw_str())])])
+            [self._get_dtype(space), space.newtext(self.raw_str())])])
         return ret
 
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -194,6 +194,9 @@
         return StringObject(obj)
     newbytes = newtext
 
+    def newunicode(self, obj):
+        raise NotImplementedError
+
     def newlist(self, items):
         return ListObject(items)
 
@@ -665,7 +668,7 @@
 
     def execute(self, interp):
         w_list = interp.space.newlist(
-            [interp.space.wrap(float(i)) for i in range(self.v)]
+            [interp.space.newfloat(float(i)) for i in range(self.v)]
         )
         dtype = get_dtype_cache(interp.space).w_float64dtype
         return array(interp.space, w_list, w_dtype=dtype, w_order=None)
@@ -832,7 +835,7 @@
                 w_res = arr.descr_take(interp.space, arg)
             elif self.name == "searchsorted":
                 w_res = arr.descr_searchsorted(interp.space, arg,
-                                               interp.space.wrap('left'))
+                                               interp.space.newtext('left'))
             else:
                 assert False # unreachable code
         elif self.name in THREE_ARG_FUNCTIONS:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -303,7 +303,7 @@
                 copy = True
             w_ret = new_view(space, orig_arr, chunks)
             if copy:
-                w_ret = w_ret.descr_copy(space, space.wrap(w_ret.get_order()))
+                w_ret = w_ret.descr_copy(space, space.newint(w_ret.get_order()))
             return w_ret
 
     def descr_setitem(self, space, orig_arr, w_index, w_value):
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -281,7 +281,7 @@
                     substr = ["(", space.str_w(space.str(
                         subdtype.subdtype.descr_get_descr(space, style='descr_subdtype'))),
                         ', ',
-                        space.str_w(space.repr(space.newtuple([space.wrap(s) for s in subdtype.shape]))),
+                        space.str_w(space.repr(space.newtuple([space.newint(s) for s in subdtype.shape]))),
                         "),"]
                 else:
                     substr = ["'", subdtype.get_str(ignore=''), "',"]
@@ -366,12 +366,12 @@
         return space.newbool(self.is_native())
 
     def descr_get_base(self, space):
-        return space.wrap(self.base)
+        return self.base
 
     def descr_get_subdtype(self, space):
         if self.subdtype is None:
             return space.w_None
-        return space.newtuple([space.wrap(self.subdtype),
+        return space.newtuple([self.subdtype,
                                self.descr_get_shape(space)])
 
     def descr_get_shape(self, space):
@@ -594,7 +594,7 @@
 
     def runpack_str(self, space, s):
         if self.is_str_or_unicode():
-            return self.coerce(space, space.wrap(s))
+            return self.coerce(space, space.newbytes(s))
         return self.itemtype.runpack_str(space, s, self.is_native())
 
     def store(self, arr, i, offset, value):
@@ -943,8 +943,8 @@
             raise
         # handle only simple cases for testing
         if space.isinstance_w(w_spec, space.w_str):
-            spec = [s.strip() for s in space.str_w(w_spec).split(',')]
-            w_lst = space.newlist([space.wrap(s) for s in spec]) 
+            spec = [s.strip() for s in space.text_w(w_spec).split(',')]
+            w_lst = space.newlist([space.newtext(s) for s in spec]) 
     if not space.isinstance_w(w_lst, space.w_list) or space.len_w(w_lst) < 1:
         raise oefmt(space.w_RuntimeError,
                     "_commastring is not returning a list with len >= 1")
@@ -1066,9 +1066,9 @@
     if space.isinstance_w(w_dtype, w_subtype):
         return w_dtype
     if space.isinstance_w(w_dtype, space.w_unicode):
-        w_dtype = space.wrap(space.str_w(w_dtype))  # may raise if invalid
-    if space.isinstance_w(w_dtype, space.w_str):
-        name = space.str_w(w_dtype)
+        w_dtype = space.newbytes(space.text_w(w_dtype))  # may raise if invalid
+    if space.isinstance_w(w_dtype, space.w_bytes):
+        name = space.bytes_w(w_dtype)
         if _check_for_commastring(name):
             return _set_metadata_and_copy(space, w_metadata,
                                 dtype_from_spec(space, w_dtype, alignment))
@@ -1454,17 +1454,17 @@
         }
         w_typeinfo = space.newdict()
         for k, v in typeinfo_partial.iteritems():
-            space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
+            space.setitem(w_typeinfo, space.newtext(k), space.gettypefor(v))
         for k, dtype in typeinfo_full.iteritems():
             itembits = dtype.elsize * 8
             if k in ('INTP', 'UINTP'):
                 char = getattr(NPY, k + 'LTR')
             else:
                 char = dtype.char
-            items_w = [space.wrap(char),
-                       space.wrap(dtype.num),
-                       space.wrap(itembits),
-                       space.wrap(dtype.itemtype.get_element_size())]
+            items_w = [space.newtext(char),
+                       space.newint(dtype.num),
+                       space.newint(itembits),
+                       space.newint(dtype.itemtype.get_element_size())]
             if dtype.is_int():
                 if dtype.is_bool():
                     w_maxobj = space.newint(1)
@@ -1478,7 +1478,7 @@
                     w_minobj = space.newint(0)
                 items_w = items_w + [w_maxobj, w_minobj]
             items_w = items_w + [dtype.w_box_type]
-            space.setitem(w_typeinfo, space.wrap(k), space.newtuple(items_w))
+            space.setitem(w_typeinfo, space.newtext(k), space.newtuple(items_w))
         self.w_typeinfo = w_typeinfo
 
 
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -40,14 +40,14 @@
         self.implementation = FakeArrayImplementation(self.base)
 
     def descr_base(self, space):
-        return space.wrap(self.base)
+        return self.base
 
     def descr_index(self, space):
-        return space.wrap(self.state.index)
+        return space.newint(self.state.index)
 
     def descr_coords(self, space):
         coords = self.iter.indices(self.state)
-        return space.newtuple([space.wrap(c) for c in coords])
+        return space.newtuple([space.newint(c) for c in coords])
 
     def descr_iter(self):
         return self
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -462,7 +462,7 @@
     signed = True
 
     def to_builtin_type(self, space, box):
-        return space.newint(self.for_computation(self.unbox(box)))
+        return space.wrap(self.for_computation(self.unbox(box)))
 
     def _base_coerce(self, space, w_item):
         if w_item is None:
@@ -740,7 +740,7 @@
     strlen = 32
 
     def to_builtin_type(self, space, box):
-        return space.wrap(self.for_computation(self.unbox(box)))
+        return space.newfloat(self.for_computation(self.unbox(box)))
 
     def _coerce(self, space, w_item):
         if w_item is None:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -876,13 +876,13 @@
         w_op_dtypes = space.w_None
         for tf in need_to_cast:
             if tf:
-                w_casting = space.wrap('safe')
-                w_op_dtypes = space.newtuple([space.wrap(d) for d in dtypes])
-                
+                w_casting = space.newtext('safe')
+                w_op_dtypes = space.newtuple([d for d in dtypes])
+
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
-        w_ro = space.newtuple([space.wrap('readonly'), space.wrap('copy')])
-        w_rw = space.newtuple([space.wrap('readwrite'), space.wrap('updateifcopy')])
-        
+        w_ro = space.newtuple([space.newtext('readonly'), space.newtext('copy')])
+        w_rw = space.newtuple([space.newtext('readwrite'), space.newtext('updateifcopy')])
+
         w_op_flags = space.newtuple([w_ro] * len(inargs) + [w_rw] * len(outargs))
         w_op_axes = space.w_None
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -90,6 +90,8 @@
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
             self._interplevel_classes[w_type] = cls
+        self.w_bytes = self.w_str
+        self.w_text = self.w_str # this is w_unicode on Py3
         self.w_dict.flag_map_or_seq = 'M'
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis

From pypy.commits at gmail.com  Mon Nov 21 05:47:24 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 02:47:24 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: merge
Message-ID: <5832d0bc.d7092e0a.c9718.41db@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88511:ffa1b618069f
Date: 2016-11-20 13:31 +0100
http://bitbucket.org/pypy/pypy/changeset/ffa1b618069f/

Log:	merge

diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -55,7 +55,7 @@
 
 eci = ExternalCompilationInfo(
     separate_module_files=[srcpath.join("cintcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
+    include_dirs=[incpath, translator_c_dir] + rootincpath,
     includes=["cintcwrapper.h"],
     library_dirs=rootlibpath,
     libraries=["Hist", "Core", "Cint"],
@@ -167,7 +167,7 @@
         memory_regulator.register(cppself)
 
         # tie all the life times to the TF1 instance
-        space.setattr(w_self, space.wrap('_callback'), w_callback)
+        space.setattr(w_self, space.newtext('_callback'), w_callback)
 
         # by definition for __init__
         return None
@@ -249,9 +249,9 @@
 def activate_branch(space, w_branch):
     w_branches = space.call_method(w_branch, "GetListOfBranches")
     for i in range(space.r_longlong_w(space.call_method(w_branches, "GetEntriesFast"))):
-        w_b = space.call_method(w_branches, "At", space.wrap(i))
+        w_b = space.call_method(w_branches, "At", space.newlong(i))
         activate_branch(space, w_b)
-    space.call_method(w_branch, "SetStatus", space.wrap(1))
+    space.call_method(w_branch, "SetStatus", space.newint(1))
     space.call_method(w_branch, "ResetReadEntry")
 
 c_ttree_GetEntry = rffi.llexternal(
@@ -277,7 +277,7 @@
 
     # try the saved cdata (for builtin types)
     try:
-        w_cdata = space.getattr(w_self, space.wrap('_'+attr))
+        w_cdata = space.getattr(w_self, space.newtext('_'+attr))
         from pypy.module._cffi_backend import cdataobj
         cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
         return cdata.convert_to_object()
@@ -302,15 +302,15 @@
         klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname))
         w_obj = klass.construct()
         # 0x10000 = kDeleteObject; reset because we own the object
-        space.call_method(w_branch, "ResetBit", space.wrap(0x10000))
+        space.call_method(w_branch, "ResetBit", space.newint(0x10000))
         space.call_method(w_branch, "SetObject", w_obj)
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
+        space.call_method(w_branch, "GetEntry", space.newlong(entry))
         space.setattr(w_self, args_w[0], w_obj)
         return w_obj
     else:
         # builtin data
         w_leaf = space.call_method(w_self, "GetLeaf", args_w[0])
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
+        space.call_method(w_branch, "GetEntry", space.newlong(entry))
 
         # location
         w_address = space.call_method(w_leaf, "GetValuePointer")
@@ -329,7 +329,7 @@
         cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename))
 
         # cache result
-        space.setattr(w_self, space.wrap('_'+attr), space.wrap(cdata))
+        space.setattr(w_self, space.newtext('_'+attr), space.wrap(cdata))
         return space.getattr(w_self, args_w[0])
 
 class W_TTreeIter(W_Root):
@@ -343,7 +343,7 @@
         self.maxentry = space.r_longlong_w(space.call_method(w_tree, "GetEntriesFast"))
 
         space = self.space = tree.space          # holds the class cache in State
-        space.call_method(w_tree, "SetBranchStatus", space.wrap("*"), space.wrap(0))
+        space.call_method(w_tree, "SetBranchStatus", space.newtext("*"), space.newint(0))
 
     def iter_w(self):
         return self.space.wrap(self)
@@ -397,7 +397,7 @@
         _method_alias(space, w_pycppclass, "__len__", "GetSize")
 
     elif name == "TF1":
-        space.setattr(w_pycppclass, space.wrap("__init__"), _pythonizations["tf1_tf1"])
+        space.setattr(w_pycppclass, space.newtext("__init__"), _pythonizations["tf1_tf1"])
 
     elif name == "TFile":
         _method_alias(space, w_pycppclass, "__getattr__", "Get")
@@ -415,9 +415,9 @@
     elif name == "TTree":
         _method_alias(space, w_pycppclass, "_unpythonized_Branch", "Branch")
 
-        space.setattr(w_pycppclass, space.wrap("Branch"),      _pythonizations["ttree_Branch"])
-        space.setattr(w_pycppclass, space.wrap("__iter__"),    _pythonizations["ttree_iter"])
-        space.setattr(w_pycppclass, space.wrap("__getattr__"), _pythonizations["ttree_getattr"])
+        space.setattr(w_pycppclass, space.newtext("Branch"),      _pythonizations["ttree_Branch"])
+        space.setattr(w_pycppclass, space.newtext("__iter__"),    _pythonizations["ttree_iter"])
+        space.setattr(w_pycppclass, space.newtext("__getattr__"), _pythonizations["ttree_getattr"])
 
     elif name[0:8] == "TVectorT":    # TVectorT<> template
         _method_alias(space, w_pycppclass, "__len__", "GetNoElements")
diff --git a/pypy/module/cppyy/capi/reflex_capi.py b/pypy/module/cppyy/capi/reflex_capi.py
--- a/pypy/module/cppyy/capi/reflex_capi.py
+++ b/pypy/module/cppyy/capi/reflex_capi.py
@@ -9,6 +9,10 @@
 srcpath = pkgpath.join("src")
 incpath = pkgpath.join("include")
 
+# require local translator path to pickup common defs
+from rpython.translator import cdir
+translator_c_dir = py.path.local(cdir)
+
 import commands
 (config_stat, incdir) = commands.getstatusoutput("root-config --incdir")
 
@@ -39,7 +43,7 @@
 
 eci = ExternalCompilationInfo(
     separate_module_files=[srcpath.join("reflexcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
+    include_dirs=[incpath, translator_c_dir] + rootincpath,
     includes=["reflexcwrapper.h"],
     library_dirs=rootlibpath,
     libraries=["Reflex"],
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -180,7 +180,7 @@
         self.size = sys.maxint
 
     def convert_argument(self, space, w_obj, address, call_local):
-        w_tc = space.findattr(w_obj, space.wrap('typecode'))
+        w_tc = space.findattr(w_obj, space.newtext('typecode'))
         if w_tc is not None and space.str_w(w_tc) != self.typecode:
             raise oefmt(space.w_TypeError,
                         "expected %s pointer type, but received %s",
@@ -312,7 +312,7 @@
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
-        return space.wrap(address[0])
+        return space.newbytes(address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
@@ -331,7 +331,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
-        return space.wrap(float(rffiptr[0]))
+        return space.newfloat(float(rffiptr[0]))
 
 class ConstFloatRefConverter(FloatConverter):
     _immutable_fields_ = ['libffitype', 'typecode']
@@ -370,7 +370,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
-        return space.wrap(rffi.charp2str(charpptr[0]))
+        return space.newbytes(rffi.charp2str(charpptr[0]))
 
     def free_argument(self, space, arg, call_local):
         lltype.free(rffi.cast(rffi.CCHARPP, arg)[0], flavor='raw')
@@ -622,7 +622,7 @@
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
-        return space.wrap(longptr[0])
+        return space.newlong(longptr[0])
 
 
 _converters = {}         # builtin and custom types
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -54,7 +54,7 @@
             raise NotImplementedError
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ptrval = rffi.cast(rffi.ULONG, lresult)
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap(self.typecode)))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext(self.typecode)))
         if ptrval == 0:
             from pypy.module.cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
@@ -128,9 +128,9 @@
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ccpresult = rffi.cast(rffi.CCHARP, lresult)
         if ccpresult == rffi.cast(rffi.CCHARP, 0):
-            return space.wrap("")
+            return space.newbytes("")
         result = rffi.charp2str(ccpresult)   # TODO: make it a choice to free
-        return space.wrap(result)
+        return space.newbytes(result)
 
 
 class ConstructorExecutor(FunctionExecutor):
@@ -139,7 +139,7 @@
         from pypy.module.cppyy import interp_cppyy
         newthis = capi.c_constructor(space, cppmethod, cpptype, num_args, args)
         assert lltype.typeOf(newthis) == capi.C_OBJECT
-        return space.wrap(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
+        return space.newlong(rffi.cast(rffi.LONG, newthis))   # really want ptrdiff_t here
 
 
 class InstancePtrExecutor(FunctionExecutor):
@@ -196,7 +196,7 @@
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         cstr_result = capi.c_call_s(space, cppmethod, cppthis, num_args, args)
-        return space.wrap(capi.charp2str_free(space, cstr_result))
+        return space.newbytes(capi.charp2str_free(space, cstr_result))
 
     def execute_libffi(self, space, cif_descr, funcaddr, buffer):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -26,7 +26,7 @@
         return arg
 
     def _wrap_object(self, space, obj):
-        return space.wrap(bool(ord(rffi.cast(rffi.CHAR, obj))))
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
 
 class CharTypeMixin(object):
     _mixin_     = True
@@ -153,7 +153,7 @@
         return r_singlefloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(float(obj))
+        return space.newfloat(float(obj))
 
 class DoubleTypeMixin(object):
     _mixin_     = True
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -33,7 +33,7 @@
     try:
         cdll = capi.c_load_dictionary(name)
     except rdynload.DLOpenError as e:
-        raise OperationError(space.w_RuntimeError, space.wrap(str(e.msg)))
+        raise OperationError(space.w_RuntimeError, space.newtext(str(e.msg)))
     return W_CPPLibrary(space, cdll)
 
 class State(object):
@@ -53,7 +53,7 @@
     if state.w_nullptr is None:
         from pypy.module._rawffi.interp_rawffi import unpack_simple_shape
         from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap('P')))
+        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext('P')))
         # TODO: fix this hack; fromaddress() will allocate memory if address
         # is null and there seems to be no way around it (ll_buffer can not
         # be touched directly)
@@ -65,7 +65,7 @@
 
 @unwrap_spec(name=str)
 def resolve_name(space, name):
-    return space.wrap(capi.c_resolve_name(space, name))
+    return space.newtext(capi.c_resolve_name(space, name))
 
 @unwrap_spec(name=str)
 def scope_byname(space, name):
@@ -113,7 +113,7 @@
     return None
 
 def std_string_name(space):
-    return space.wrap(capi.std_string_name)
+    return space.newtext(capi.std_string_name)
 
 @unwrap_spec(w_callback=W_Root)
 def set_class_generator(space, w_callback):
@@ -126,7 +126,7 @@
     state.w_fngen_callback = w_callback
 
 def register_class(space, w_pycppclass):
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
@@ -431,7 +431,7 @@
             try:
                 s = self.space.str_w(args_w[i])
             except OperationError:
-                s = self.space.str_w(self.space.getattr(args_w[i], self.space.wrap('__name__')))
+                s = self.space.str_w(self.space.getattr(args_w[i], self.space.newtext('__name__')))
             s = capi.c_resolve_name(self.space, s)
             if s != self.templ_args[i]:
                 raise oefmt(self.space.w_TypeError,
@@ -537,7 +537,7 @@
         # only get here if all overloads failed ...
         errmsg = 'none of the %d overloaded methods succeeded. Full details:' % len(self.functions)
         if hasattr(self.space, "fake"):     # FakeSpace fails errorstr (see below)
-            raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+            raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
@@ -554,13 +554,13 @@
                 errmsg += '\n  '+cppyyfunc.signature()+' =>\n'
                 errmsg += '    Exception: '+str(e)
 
-        raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
+        raise OperationError(self.space.w_TypeError, self.space.newtext(errmsg))
 
     def signature(self):
         sig = self.functions[0].signature()
         for i in range(1, len(self.functions)):
             sig += '\n'+self.functions[i].signature()
-        return self.space.wrap(sig)
+        return self.space.newtext(sig)
 
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.signature() for f in self.functions]
@@ -633,7 +633,7 @@
         self.offset = offset
 
     def get_returntype(self):
-        return self.space.wrap(self.converter.name)
+        return self.space.newtext(self.converter.name)
 
     def _get_offset(self, cppinstance):
         if cppinstance:
@@ -750,7 +750,7 @@
         return capi.c_scoped_final_name(self.space, self.handle)
 
     def get_method_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.methods])
+        return self.space.newlist([self.space.newtext(name) for name in self.methods])
 
     def get_overload(self, name):
         try:
@@ -762,7 +762,7 @@
         return new_method
 
     def get_datamember_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.datamembers])
+        return self.space.newlist([self.space.newtext(name) for name in self.datamembers])
 
     def get_datamember(self, name):
         try:
@@ -856,17 +856,17 @@
         alldir = []
         for i in range(capi.c_num_scopes(self.space, self)):
             sname = capi.c_scope_name(self.space, self, i)
-            if sname: alldir.append(self.space.wrap(sname))
+            if sname: alldir.append(self.space.newtext(sname))
         allmeth = {}
         for i in range(capi.c_num_methods(self.space, self)):
             idx = capi.c_method_index_at(self.space, self, i)
             mname = capi.c_method_name(self.space, self, idx)
             if mname: allmeth.setdefault(mname, 0)
         for m in allmeth.keys():
-            alldir.append(self.space.wrap(m))
+            alldir.append(self.space.newtext(m))
         for i in range(capi.c_num_datamembers(self.space, self)):
             dname = capi.c_datamember_name(self.space, self, i)
-            if dname: alldir.append(self.space.wrap(dname))
+            if dname: alldir.append(self.space.newtext(dname))
         return self.space.newlist(alldir)
         
 
@@ -952,7 +952,7 @@
         num_bases = capi.c_num_bases(self.space, self)
         for i in range(num_bases):
             base_name = capi.c_base_name(self.space, self, i)
-            bases.append(self.space.wrap(base_name))
+            bases.append(self.space.newtext(base_name))
         return self.space.newlist(bases)
 
 W_CPPClass.typedef = TypeDef(
@@ -1038,7 +1038,8 @@
         self._opt_register_finalizer()
 
     def _opt_register_finalizer(self):
-        if self.python_owns and not self.finalizer_registered:
+        if self.python_owns and not self.finalizer_registered \
+               and not hasattr(self.space, "fake"):
             self.register_finalizer(self.space)
             self.finalizer_registered = True
 
@@ -1049,7 +1050,7 @@
 
     # allow user to determine ownership rules on a per object level
     def fget_python_owns(self, space):
-        return space.wrap(self.python_owns)
+        return space.newbool(self.python_owns)
 
     @unwrap_spec(value=bool)
     def fset_python_owns(self, space, value):
@@ -1091,7 +1092,7 @@
     def instance__eq__(self, w_other):
         # special case: if other is None, compare pointer-style
         if self.space.is_w(w_other, self.space.w_None):
-            return self.space.wrap(not self._rawobject)
+            return self.space.newbool(not self._rawobject)
 
         # get here if no class-specific overloaded operator is available, try to
         # find a global overload in gbl, in __gnu_cxx (for iterators), or in the
@@ -1121,7 +1122,7 @@
         # the first data member in a struct and the struct have the same address)
         other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)  # TODO: factor out
         iseq = (self._rawobject == other._rawobject) and (self.cppclass == other.cppclass)
-        return self.space.wrap(iseq)
+        return self.space.newbool(iseq)
 
     def instance__ne__(self, w_other):
         return self.space.not_(self.instance__eq__(w_other))
@@ -1149,7 +1150,7 @@
         w_as_builtin = self._get_as_builtin()
         if w_as_builtin is not None:
             return self.space.repr(w_as_builtin)
-        return self.space.wrap("<%s object at 0x%x>" %
+        return self.space.newtext("<%s object at 0x%x>" %
                                (self.cppclass.name, rffi.cast(rffi.ULONG, self.get_rawobject())))
 
     def destruct(self):
@@ -1215,12 +1216,12 @@
     except KeyError:
         final_name = capi.c_scoped_final_name(space, handle)
         # the callback will cache the class by calling register_class
-        w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name))
+        w_pycppclass = space.call_function(state.w_clgen_callback, space.newtext(final_name))
     return w_pycppclass
 
 def get_interface_func(space, w_callable, npar):
     state = space.fromcache(State)
-    return space.call_function(state.w_fngen_callback, w_callable, space.wrap(npar))
+    return space.call_function(state.w_fngen_callback, w_callable, space.newint(npar))
 
 def wrap_cppobject(space, rawobject, cppclass,
                    do_cast=True, python_owns=False, is_ref=False, fresh=False):
@@ -1235,7 +1236,7 @@
                 w_pycppclass = get_pythonized_cppclass(space, actual)
                 offset = capi.c_base_offset1(space, actual, cppclass, rawobject, -1)
                 rawobject = capi.direct_ptradd(rawobject, offset)
-                w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+                w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
                 cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
             except Exception:
                 # failed to locate/build the derived class, so stick to the base (note
@@ -1272,7 +1273,7 @@
 def addressof(space, w_obj):
     """Takes a bound C++ instance or array, returns the raw address."""
     address = _addressof(space, w_obj)
-    return space.wrap(address)
+    return space.newlong(address)
 
 @unwrap_spec(owns=bool, cast=bool)
 def bind_object(space, w_obj, w_pycppclass, owns=False, cast=False):
@@ -1283,7 +1284,7 @@
     except Exception:
         # accept integer value as address
         rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj))
-    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    w_cppclass = space.findattr(w_pycppclass, space.newtext("_cpp_proxy"))
     if not w_cppclass:
         w_cppclass = scope_byname(space, space.str_w(w_pycppclass))
         if not w_cppclass:
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -18,8 +18,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct = cls.space.wrap(test_dct)
-        cls.w_capi_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct = cls.space.newtext(test_dct)
+        cls.w_capi_identity = cls.space.newtext(capi.identify())
         cls.w_advanced = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py
--- a/pypy/module/cppyy/test/test_cint.py
+++ b/pypy/module/cppyy/test/test_cint.py
@@ -169,11 +169,11 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)
-        cls.w_M = cls.space.wrap(10)
-        cls.w_fname = cls.space.wrap("test.root")
-        cls.w_tname = cls.space.wrap("test")
-        cls.w_title = cls.space.wrap("test tree")
+        cls.w_N = cls.space.newint(5)
+        cls.w_M = cls.space.newint(10)
+        cls.w_fname = cls.space.newtext("test.root")
+        cls.w_tname = cls.space.newtext("test")
+        cls.w_title = cls.space.newtext("test tree")
         cls.w_iotypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (iotypes_dct,))
diff --git a/pypy/module/cppyy/test/test_crossing.py b/pypy/module/cppyy/test/test_crossing.py
--- a/pypy/module/cppyy/test/test_crossing.py
+++ b/pypy/module/cppyy/test/test_crossing.py
@@ -9,8 +9,6 @@
 from pypy.module.cpyext import api
 from pypy.module.cpyext.state import State
 
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("crossingDict.so"))
@@ -24,7 +22,7 @@
 
 # from pypy/module/cpyext/test/test_cpyext.py; modified to accept more external
 # symbols and called directly instead of import_module
-def compile_extension_module(space, modname, symbols, **kwds):
+def compile_extension_module(space, modname, **kwds):
     """
     Build an extension module and return the filename of the resulting native
     code file.
@@ -38,19 +36,23 @@
     state = space.fromcache(State)
     api_library = state.api_lib
     if sys.platform == 'win32':
-        kwds["libraries"] = [api_library]
+        kwds["libraries"] = []#[api_library]
         # '%s' undefined; assuming extern returning int
         kwds["compile_extra"] = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
+                              (space.int_w(w_maj), space.int_w(w_min))]
     elif sys.platform == 'darwin':
         kwds["link_files"] = [str(api_library + '.dylib')]
     else:
         kwds["link_files"] = [str(api_library + '.so')]
         if sys.platform.startswith('linux'):
-            kwds["compile_extra"]=["-Werror=implicit-function-declaration"]
+            kwds["compile_extra"]=["-Werror", "-g", "-O0"]
+            kwds["link_extra"]=["-g"]
 
     modname = modname.split('.')[-1]
     eci = ExternalCompilationInfo(
-        #export_symbols=['init%s' % (modname,)]+symbols,
         include_dirs=api.include_dirs,
         **kwds
         )
@@ -65,28 +67,30 @@
     soname.rename(pydname)
     return str(pydname)
 
-class AppTestCrossing(AppTestCpythonExtensionBase):
-    spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools', 'cpyext'])
+class AppTestCrossing:
+    spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        AppTestCpythonExtensionBase.setup_class.im_func(cls)
         # cppyy specific additions (note that test_dct is loaded late
         # to allow the generated extension module be loaded first)
-        cls.w_test_dct    = cls.space.wrap(test_dct)
+        cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
-            import cppyy, cpyext, ctypes""")    # prevents leak-checking complaints on ctypes
+            import ctypes, cppyy""")    # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
-        AppTestCpythonExtensionBase.setup_method.im_func(self, func)
-
         @unwrap_spec(name=str, init=str, body=str)
-        def create_cdll(space, name, init, body, w_symbols):
+        def create_cdll(space, name, init, body):
             # the following is loosely from test_cpyext.py import_module; it
             # is copied here to be able to tweak the call to
             # compile_extension_module and to get a different return result
             # than in that function
             code = """
             #include <Python.h>
+            /* fix for cpython 2.7 Python.h if running tests with -A
+               since pypy compiles with -fvisibility-hidden */
+            #undef PyMODINIT_FUNC
+            #define PyMODINIT_FUNC RPY_EXPORTED void
+
             %(body)s
 
             PyMODINIT_FUNC
@@ -95,28 +99,16 @@
             }
             """ % dict(name=name, init=init, body=body)
             kwds = dict(separate_module_sources=[code])
-            symbols = [space.str_w(w_item) for w_item in space.fixedview(w_symbols)]
-            mod = compile_extension_module(space, name, symbols, **kwds)
+            mod = compile_extension_module(space, name, **kwds)
 
             # explicitly load the module as a CDLL rather than as a module
             from pypy.module.imp.importing import get_so_extension
             fullmodname = os.path.join(
                 os.path.dirname(mod), name + get_so_extension(space))
-            return space.wrap(fullmodname)
+            return space.newtext(fullmodname)
 
         self.w_create_cdll = self.space.wrap(interp2app(create_cdll))
 
-    def test00_base_class(self):
-        """Test from cpyext; only here to see whether the imported class works"""
-
-        import sys
-        init = """
-        if (Py_IsInitialized())
-            Py_InitModule("foo", NULL);
-        """
-        self.import_module(name='foo', init=init)
-        assert 'foo' in sys.modules
-
     def test01_build_bar_extension(self):
         """Test that builds the needed extension; runs as test to keep it loaded"""
 
@@ -131,10 +123,12 @@
 
         # note: only the symbols are needed for C, none for python
         body = """
+        RPY_EXPORTED
         long bar_unwrap(PyObject* arg)
         {
-            return PyLong_AsLong(arg);
+            return 13;//PyLong_AsLong(arg);
         }
+        RPY_EXPORTED
         PyObject* bar_wrap(long l)
         {
             return PyLong_FromLong(l);
@@ -146,8 +140,7 @@
         # explicitly load the module as a CDLL rather than as a module
         import ctypes
         self.cmodule = ctypes.CDLL(
-            self.create_cdll(name, init, body, ['bar_unwrap', 'bar_wrap']),
-            ctypes.RTLD_GLOBAL)
+            self.create_cdll(name, init, body), ctypes.RTLD_GLOBAL)
 
     def test02_crossing_dict(self):
         """Test availability of all needed classes in the dict"""
@@ -160,6 +153,7 @@
 
         assert crossing.A == crossing.A
 
+    @py.test.mark.dont_track_allocations("fine when running standalone, though?!")
     def test03_send_pyobject(self):
         """Test sending a true pyobject to C++"""
 
@@ -169,6 +163,7 @@
         a = crossing.A()
         assert a.unwrap(13) == 13
 
+    @py.test.mark.dont_track_allocations("fine when running standalone, though?!")
     def test04_send_and_receive_pyobject(self):
         """Test receiving a true pyobject from C++"""
 
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)  # should be imported from the dictionary
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -17,8 +17,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
-        cls.w_identity = cls.space.wrap(capi.identify())
+        cls.w_test_dct  = cls.space.newtext(test_dct)
+        cls.w_identity = cls.space.newtext(capi.identify())
         cls.w_fragile = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_operators.py b/pypy/module/cppyy/test/test_operators.py
--- a/pypy/module/cppyy/test/test_operators.py
+++ b/pypy/module/cppyy/test/test_operators.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(5)  # should be imported from the dictionary
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_overloads.py b/pypy/module/cppyy/test/test_overloads.py
--- a/pypy/module/cppyy/test/test_overloads.py
+++ b/pypy/module/cppyy/test/test_overloads.py
@@ -16,7 +16,7 @@
 
     def setup_class(cls):
         env = os.environ
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -17,7 +17,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -376,7 +376,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -15,8 +15,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -203,7 +203,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -279,8 +279,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -335,8 +335,8 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_N = cls.space.wrap(13)
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_N = cls.space.newint(13)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -444,7 +444,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -481,7 +481,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlstring = cls.space.appexec([], """():
             import cppyy, sys
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_streams.py b/pypy/module/cppyy/test/test_streams.py
--- a/pypy/module/cppyy/test/test_streams.py
+++ b/pypy/module/cppyy/test/test_streams.py
@@ -15,7 +15,7 @@
     spaceconfig = dict(usemodules=['cppyy', '_rawffi', 'itertools'])
 
     def setup_class(cls):
-        cls.w_test_dct  = cls.space.wrap(test_dct)
+        cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_streams = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -3,14 +3,17 @@
 from rpython.rlib.objectmodel import specialize, instantiate
 from rpython.rlib import rarithmetic, jit
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper import llinterp
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
 
-from pypy.module.cppyy import interp_cppyy, capi
+from pypy.module.cppyy import interp_cppyy, capi, executor
 # These tests are for the backend that support the fast path only.
 if capi.identify() == 'CINT':
     py.test.skip("CINT does not support fast path")
 elif capi.identify() == 'loadable_capi':
     py.test.skip("can not currently use FakeSpace with _cffi_backend")
+elif os.getenv("CPPYY_DISABLE_FASTPATH"):
+    py.test.skip("fast path is disabled by CPPYY_DISABLE_FASTPATH envar")
 
 # load cpyext early, or its global vars are counted as leaks in the test
 # (note that the module is not otherwise used in the test itself)
@@ -29,6 +32,23 @@
     return rffi.ptradd(ptr, offset)
 capi.exchange_address = _opaque_exchange_address
 
+# add missing alt_errno (??)
+def get_tlobj(self):
+    try:
+        return self._tlobj
+    except AttributeError:
+        from rpython.rtyper.lltypesystem import rffi
+        PERRNO = rffi.CArrayPtr(rffi.INT)
+        fake_p_errno = lltype.malloc(PERRNO.TO, 1, flavor='raw', zero=True,
+                                     track_allocation=False)
+        self._tlobj = {'RPY_TLOFS_p_errno': fake_p_errno,
+                       'RPY_TLOFS_alt_errno': rffi.cast(rffi.INT, 0),
+                       #'thread_ident': ...,
+                       }
+        return self._tlobj
+llinterp.LLInterpreter.get_tlobj = get_tlobj
+
+
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("example01Dict.so"))
 
@@ -43,6 +63,10 @@
 class FakeBase(W_Root):
     typename = None
 
+class FakeBool(FakeBase):
+    typename = "bool"
+    def __init__(self, val):
+        self.val = val
 class FakeInt(FakeBase):
     typename = "int"
     def __init__(self, val):
@@ -69,9 +93,10 @@
     def get_raw_address(self):
         raise ValueError("no raw buffer")
 class FakeException(FakeType):
-    def __init__(self, name):
+    def __init__(self, space, name):
         FakeType.__init__(self, name)
-        self.message = name
+        self.msg = name
+        self.space = space
 
 class FakeUserDelAction(object):
     def __init__(self, space):
@@ -83,16 +108,13 @@
     def perform(self, executioncontext, frame):
         pass
 
+class FakeState(object):
+    def __init__(self, space):
+        self.slowcalls = 0
+
 class FakeSpace(object):
     fake = True
 
-    w_ValueError = FakeException("ValueError")
-    w_TypeError = FakeException("TypeError")
-    w_AttributeError = FakeException("AttributeError")
-    w_ReferenceError = FakeException("ReferenceError")
-    w_NotImplementedError = FakeException("NotImplementedError")
-    w_RuntimeError = FakeException("RuntimeError")
-
     w_None = None
     w_str = FakeType("str")
     w_int = FakeType("int")
@@ -105,6 +127,21 @@
         self.config = dummy()
         self.config.translating = False
 
+        # kill calls to c_call_i (i.e. slow path)
+        def c_call_i(space, cppmethod, cppobject, nargs, args):
+            assert not "slow path called"
+            return capi.c_call_i(space, cppmethod, cppobject, nargs, args)
+        executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i)
+
+        self.w_AttributeError      = FakeException(self, "AttributeError")
+        self.w_KeyError            = FakeException(self, "KeyError")
+        self.w_NotImplementedError = FakeException(self, "NotImplementedError")
+        self.w_ReferenceError      = FakeException(self, "ReferenceError")
+        self.w_RuntimeError        = FakeException(self, "RuntimeError")
+        self.w_SystemError         = FakeException(self, "SystemError")
+        self.w_TypeError           = FakeException(self, "TypeError")
+        self.w_ValueError          = FakeException(self, "ValueError")
+
     def issequence_w(self, w_obj):
         return True
 
@@ -120,6 +157,30 @@
             return FakeInt(int(obj))
         assert 0
 
+    @specialize.argtype(1)
+    def newbool(self, obj):
+        return FakeBool(obj)
+
+    @specialize.argtype(1)
+    def newint(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newlong(self, obj):
+        return FakeInt(obj)
+
+    @specialize.argtype(1)
+    def newfloat(self, obj):
+        return FakeFloat(obj)
+
+    @specialize.argtype(1)
+    def newbytes(self, obj):
+        return FakeString(obj)
+
+    @specialize.argtype(1)
+    def newtext(self, obj):
+        return FakeString(obj)
+
     def float_w(self, w_obj, allow_conversion=True):
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
@@ -210,7 +271,8 @@
         f()
         space = FakeSpace()
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(1)
+        self.check_jitcell_token_count(1)   # same for fast and slow path??
+        # rely on replacement of capi calls to raise exception instead (see FakeSpace.__init__)
 
     def test01_simple(self):
         """Test fast path being taken for methods"""

From pypy.commits at gmail.com  Mon Nov 21 05:47:27 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 02:47:27 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: make interp_attrproperty take a
 required wrapfn argument (the py3.5 branch
Message-ID: <5832d0bf.cf212e0a.ebdca.4758@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88512:036370c1dfd7
Date: 2016-11-21 11:46 +0100
http://bitbucket.org/pypy/pypy/changeset/036370c1dfd7/

Log:	make interp_attrproperty take a required wrapfn argument (the py3.5
	branch already started to invent workarounds for this)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -808,6 +808,11 @@
                 raise
             return None
 
+    def wrap_none(self, w_obj):
+        if w_obj is None:
+            return self.w_None
+        return w_obj
+
     @signature(types.any(), types.bool(), returns=types.instance(W_Root))
     def newbool(self, b):
         if b:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -324,11 +324,11 @@
     def descr_get_objclass(space, property):
         return property.objclass_getter(space)
 
-def interp_attrproperty(name, cls, doc=None):
+def interp_attrproperty(name, cls, doc=None, wrapfn=None):
     "NOT_RPYTHON: initialization-time only"
-    # YYY needs some refactoring to get rid of the wrap
+    assert wrapfn is not None
     def fget(space, obj):
-        return space.wrap(getattr(obj, name))
+        return getattr(space, wrapfn)(getattr(obj, name))
     return GetSetProperty(fget, cls=cls, doc=doc)
 
 def interp_attrproperty_w(name, cls, doc=None):
@@ -347,9 +347,9 @@
     __get__ = interp2app(GetSetProperty.descr_property_get),
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
-    __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __name__ = interp_attrproperty('name', cls=GetSetProperty, wrapfn="newtext_or_none"),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
-    __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
+    __doc__ = interp_attrproperty('doc', cls=GetSetProperty, wrapfn="newtext_or_none"),
     )
 assert not GetSetProperty.typedef.acceptable_as_base_class  # no __new__
 
@@ -402,7 +402,7 @@
     __get__ = interp2app(Member.descr_member_get),
     __set__ = interp2app(Member.descr_member_set),
     __delete__ = interp2app(Member.descr_member_del),
-    __name__ = interp_attrproperty('name', cls=Member),
+    __name__ = interp_attrproperty('name', cls=Member, wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_cls', cls=Member),
     )
 assert not Member.typedef.acceptable_as_base_class  # no __new__
@@ -528,7 +528,7 @@
 
 
 Code.typedef = TypeDef('internal-code',
-    co_name = interp_attrproperty('co_name', cls=Code),
+    co_name = interp_attrproperty('co_name', cls=Code, wrapfn="newtext_or_none"),
     co_varnames = GetSetProperty(fget_co_varnames, cls=Code),
     co_argcount = GetSetProperty(fget_co_argcount, cls=Code),
     co_flags = GetSetProperty(fget_co_flags, cls=Code),
@@ -538,7 +538,7 @@
 
 BuiltinCode.typedef = TypeDef('builtin-code',
     __reduce__   = interp2app(BuiltinCode.descr__reduce__),
-    co_name = interp_attrproperty('co_name', cls=BuiltinCode),
+    co_name = interp_attrproperty('co_name', cls=BuiltinCode, wrapfn="newtext_or_none"),
     co_varnames = GetSetProperty(fget_co_varnames, cls=BuiltinCode),
     co_argcount = GetSetProperty(fget_co_argcount, cls=BuiltinCode),
     co_flags = GetSetProperty(fget_co_flags, cls=BuiltinCode),
@@ -554,20 +554,20 @@
     __hash__ = interp2app(PyCode.descr_code__hash__),
     __reduce__ = interp2app(PyCode.descr__reduce__),
     __repr__ = interp2app(PyCode.repr),
-    co_argcount = interp_attrproperty('co_argcount', cls=PyCode),
-    co_nlocals = interp_attrproperty('co_nlocals', cls=PyCode),
-    co_stacksize = interp_attrproperty('co_stacksize', cls=PyCode),
-    co_flags = interp_attrproperty('co_flags', cls=PyCode),
-    co_code = interp_attrproperty('co_code', cls=PyCode),
+    co_argcount = interp_attrproperty('co_argcount', cls=PyCode, wrapfn="newint"),
+    co_nlocals = interp_attrproperty('co_nlocals', cls=PyCode, wrapfn="newint"),
+    co_stacksize = interp_attrproperty('co_stacksize', cls=PyCode, wrapfn="newint"),
+    co_flags = interp_attrproperty('co_flags', cls=PyCode, wrapfn="newint"),
+    co_code = interp_attrproperty('co_code', cls=PyCode, wrapfn="newbytes"),
     co_consts = GetSetProperty(PyCode.fget_co_consts),
     co_names = GetSetProperty(PyCode.fget_co_names),
     co_varnames = GetSetProperty(PyCode.fget_co_varnames),
     co_freevars = GetSetProperty(PyCode.fget_co_freevars),
     co_cellvars = GetSetProperty(PyCode.fget_co_cellvars),
-    co_filename = interp_attrproperty('co_filename', cls=PyCode),
-    co_name = interp_attrproperty('co_name', cls=PyCode),
-    co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode),
-    co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode),
+    co_filename = interp_attrproperty('co_filename', cls=PyCode, wrapfn="newtext"),
+    co_name = interp_attrproperty('co_name', cls=PyCode, wrapfn="newtext"),
+    co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode, wrapfn="newint"),
+    co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode, wrapfn="newbytes"),
     __weakref__ = make_weakref_descr(PyCode),
     )
 PyCode.typedef.acceptable_as_base_class = False
@@ -732,10 +732,10 @@
 PyTraceback.typedef = TypeDef("traceback",
     __reduce__ = interp2app(PyTraceback.descr__reduce__),
     __setstate__ = interp2app(PyTraceback.descr__setstate__),
-    tb_frame = interp_attrproperty('frame', cls=PyTraceback),
-    tb_lasti = interp_attrproperty('lasti', cls=PyTraceback),
+    tb_frame = interp_attrproperty_w('frame', cls=PyTraceback),
+    tb_lasti = interp_attrproperty('lasti', cls=PyTraceback, wrapfn="newint"),
     tb_lineno = GetSetProperty(PyTraceback.descr_tb_lineno),
-    tb_next = interp_attrproperty('next', cls=PyTraceback),
+    tb_next = interp_attrproperty_w('next', cls=PyTraceback),
     )
 assert not PyTraceback.typedef.acceptable_as_base_class  # no __new__
 
@@ -753,7 +753,7 @@
                             descrmismatch='close'),
     __iter__   = interp2app(GeneratorIterator.descr__iter__,
                             descrmismatch='__iter__'),
-    gi_running = interp_attrproperty('running', cls=GeneratorIterator),
+    gi_running = interp_attrproperty('running', cls=GeneratorIterator, wrapfn="newbool"),
     gi_frame   = GetSetProperty(GeneratorIterator.descr_gi_frame),
     gi_code    = GetSetProperty(GeneratorIterator.descr_gi_code),
     __name__   = GetSetProperty(GeneratorIterator.descr__name__),
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -4,7 +4,7 @@
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty, interp_attrproperty_w
 
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
@@ -342,10 +342,10 @@
 
 W_CField.typedef = TypeDef(
     '_cffi_backend.CField',
-    type = interp_attrproperty('ctype', W_CField),
-    offset = interp_attrproperty('offset', W_CField),
-    bitshift = interp_attrproperty('bitshift', W_CField),
-    bitsize = interp_attrproperty('bitsize', W_CField),
-    flags = interp_attrproperty('flags', W_CField),
+    type = interp_attrproperty_w('ctype', W_CField),
+    offset = interp_attrproperty('offset', W_CField, wrapfn="newint"),
+    bitshift = interp_attrproperty('bitshift', W_CField, wrapfn="newint"),
+    bitsize = interp_attrproperty('bitsize', W_CField, wrapfn="newint"),
+    flags = interp_attrproperty('flags', W_CField, wrapfn="newint"),
     )
 W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -138,8 +138,8 @@
         'FFIFunctionWrapper',
         __repr__ = interp2app(W_FunctionWrapper.descr_repr),
         __call__ = interp2app(W_FunctionWrapper.descr_call),
-        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper),
-        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper),
+        __name__ = interp_attrproperty('fnname', cls=W_FunctionWrapper, wrapfn="newtext"),
+        __module__ = interp_attrproperty('modulename', cls=W_FunctionWrapper, wrapfn="newtext"),
         __doc__ = GetSetProperty(W_FunctionWrapper.descr_get_doc),
         __get__ = interp2app(W_FunctionWrapper.descr_get),
         )
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -163,14 +163,20 @@
         '_csv.Dialect',
         __new__ = interp2app(W_Dialect___new__),
 
-        delimiter        = interp_attrproperty('delimiter', W_Dialect),
-        doublequote      = interp_attrproperty('doublequote', W_Dialect),
+        delimiter        = interp_attrproperty('delimiter', W_Dialect,
+            wrapfn='newtext'),
+        doublequote      = interp_attrproperty('doublequote', W_Dialect,
+            wrapfn='newbool'),
         escapechar       = GetSetProperty(_get_escapechar, cls=W_Dialect),
-        lineterminator   = interp_attrproperty('lineterminator', W_Dialect),
+        lineterminator   = interp_attrproperty('lineterminator', W_Dialect,
+            wrapfn='newtext'),
         quotechar        = GetSetProperty(_get_quotechar, cls=W_Dialect),
-        quoting          = interp_attrproperty('quoting', W_Dialect),
-        skipinitialspace = interp_attrproperty('skipinitialspace', W_Dialect),
-        strict           = interp_attrproperty('strict', W_Dialect),
+        quoting          = interp_attrproperty('quoting', W_Dialect,
+            wrapfn='newint'),
+        skipinitialspace = interp_attrproperty('skipinitialspace', W_Dialect,
+            wrapfn='newbool'),
+        strict           = interp_attrproperty('strict', W_Dialect,
+            wrapfn='newbool'),
 
         __doc__ = """CSV dialect
 
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -247,7 +247,8 @@
 W_Reader.typedef = TypeDef(
         '_csv.reader',
         dialect = interp_attrproperty_w('dialect', W_Reader),
-        line_num = interp_attrproperty('line_num', W_Reader),
+        line_num = interp_attrproperty('line_num', W_Reader,
+            wrapfn="newint"),
         __iter__ = interp2app(W_Reader.iter_w),
         next = interp2app(W_Reader.next_w),
         __doc__ = """CSV reader
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -613,9 +613,12 @@
     name     = interp_attrproperty_w('w_name', cls=W_File, doc="file name"),
     mode     = interp_attrproperty('mode', cls=W_File,
                               doc = "file mode ('r', 'U', 'w', 'a', "
-                                    "possibly with 'b' or '+' added)"),
-    encoding = interp_attrproperty('encoding', cls=W_File),
-    errors = interp_attrproperty('errors', cls=W_File),
+                                    "possibly with 'b' or '+' added)",
+                                    wrapfn="newtext"),
+    encoding = interp_attrproperty('encoding', cls=W_File,
+        wrapfn="wrap_none"),
+    errors = interp_attrproperty('errors', cls=W_File,
+        wrapfn="wrap_none"),
     closed   = GetSetProperty(descr_file_closed, cls=W_File,
                               doc="True if the file is closed"),
     newlines = GetSetProperty(descr_file_newlines, cls=W_File,
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -449,7 +449,8 @@
     fileno = interp2app(W_FileIO.fileno_w),
     isatty = interp2app(W_FileIO.isatty_w),
     name = interp_member_w('w_name', cls=W_FileIO),
-    closefd = interp_attrproperty('closefd', cls=W_FileIO),
+    closefd = interp_attrproperty('closefd', cls=W_FileIO,
+        wrapfn="newbool"),
     mode = GetSetProperty(W_FileIO.descr_get_mode),
     )
 
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -34,7 +34,8 @@
                "I/O stream"),
     __new__  = generic_new_descr(W_BlockingIOError),
     __init__ = interp2app(W_BlockingIOError.descr_init),
-    characters_written = interp_attrproperty('written', W_BlockingIOError),
+    characters_written = interp_attrproperty('written', W_BlockingIOError,
+        wrapfn="newint"),
     )
 
 DEFAULT_BUFFER_SIZE = 8 * 1024
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -1045,7 +1045,8 @@
     truncate = interp2app(W_TextIOWrapper.truncate_w),
     close = interp2app(W_TextIOWrapper.close_w),
 
-    line_buffering = interp_attrproperty("line_buffering", W_TextIOWrapper),
+    line_buffering = interp_attrproperty("line_buffering", W_TextIOWrapper,
+        wrapfn="newint"),
     readable = interp2app(W_TextIOWrapper.readable_w),
     writable = interp2app(W_TextIOWrapper.writable_w),
     seekable = interp2app(W_TextIOWrapper.seekable_w),
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -64,10 +64,14 @@
 W_StatsEntry.typedef = TypeDef(
     'StatsEntry',
     code = GetSetProperty(W_StatsEntry.get_code),
-    callcount = interp_attrproperty('callcount', W_StatsEntry),
-    reccallcount = interp_attrproperty('reccallcount', W_StatsEntry),
-    inlinetime = interp_attrproperty('it', W_StatsEntry),
-    totaltime = interp_attrproperty('tt', W_StatsEntry),
+    callcount = interp_attrproperty('callcount', W_StatsEntry,
+        wrapfn="newint"),
+    reccallcount = interp_attrproperty('reccallcount', W_StatsEntry,
+        wrapfn="newint"),
+    inlinetime = interp_attrproperty('it', W_StatsEntry,
+        wrapfn="newfloat"),
+    totaltime = interp_attrproperty('tt', W_StatsEntry,
+        wrapfn="newfloat"),
     calls = GetSetProperty(W_StatsEntry.get_calls),
     __repr__ = interp2app(W_StatsEntry.repr),
 )
@@ -91,10 +95,14 @@
 W_StatsSubEntry.typedef = TypeDef(
     'SubStatsEntry',
     code = GetSetProperty(W_StatsSubEntry.get_code),
-    callcount = interp_attrproperty('callcount', W_StatsSubEntry),
-    reccallcount = interp_attrproperty('reccallcount', W_StatsSubEntry),
-    inlinetime = interp_attrproperty('it', W_StatsSubEntry),
-    totaltime = interp_attrproperty('tt', W_StatsSubEntry),
+    callcount = interp_attrproperty('callcount', W_StatsSubEntry,
+        wrapfn="newint"),
+    reccallcount = interp_attrproperty('reccallcount', W_StatsSubEntry,
+        wrapfn="newint"),
+    inlinetime = interp_attrproperty('it', W_StatsSubEntry,
+        wrapfn="newfloat"),
+    totaltime = interp_attrproperty('tt', W_StatsSubEntry,
+        wrapfn="newfloat"),
     __repr__ = interp2app(W_StatsSubEntry.repr),
 )
 
diff --git a/pypy/module/_rawffi/alt/interp_ffitype.py b/pypy/module/_rawffi/alt/interp_ffitype.py
--- a/pypy/module/_rawffi/alt/interp_ffitype.py
+++ b/pypy/module/_rawffi/alt/interp_ffitype.py
@@ -105,7 +105,8 @@
 
 W_FFIType.typedef = TypeDef(
     'FFIType',
-    name = interp_attrproperty('name', W_FFIType),
+    name = interp_attrproperty('name', W_FFIType,
+        wrapfn="newtext_or_none"),
     __repr__ = interp2app(W_FFIType.repr),
     deref_pointer = interp2app(W_FFIType.descr_deref_pointer),
     sizeof = interp2app(W_FFIType.descr_sizeof),
diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py
--- a/pypy/module/_rawffi/alt/interp_struct.py
+++ b/pypy/module/_rawffi/alt/interp_struct.py
@@ -5,7 +5,7 @@
 from rpython.rlib.rgc import must_be_light_finalizer
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import oefmt
 from pypy.module._rawffi.alt.interp_ffitype import W_FFIType
@@ -29,9 +29,11 @@
 W_Field.typedef = TypeDef(
     'Field',
     __new__ = interp2app(descr_new_field),
-    name = interp_attrproperty('name', W_Field),
-    ffitype = interp_attrproperty('w_ffitype', W_Field),
-    offset = interp_attrproperty('offset', W_Field),
+    name = interp_attrproperty('name', W_Field,
+        wrapfn="newtext_or_none"),
+    ffitype = interp_attrproperty_w('w_ffitype', W_Field),
+    offset = interp_attrproperty('offset', W_Field,
+        wrapfn="newint"),
     )
 
 
@@ -141,7 +143,7 @@
 W__StructDescr.typedef = TypeDef(
     '_StructDescr',
     __new__ = interp2app(descr_new_structdescr),
-    ffitype = interp_attrproperty('w_ffitype', W__StructDescr),
+    ffitype = interp_attrproperty_w('w_ffitype', W__StructDescr),
     define_fields = interp2app(W__StructDescr.define_fields),
     allocate = interp2app(W__StructDescr.allocate),
     fromaddress = interp2app(W__StructDescr.fromaddress),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -4,7 +4,7 @@
 """
 
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._rawffi.interp_rawffi import segfault_exception
@@ -199,7 +199,7 @@
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
     __len__     = interp2app(W_ArrayInstance.getlength),
     buffer      = GetSetProperty(W_ArrayInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_ArrayInstance),
+    shape       = interp_attrproperty_w('shape', W_ArrayInstance),
     free        = interp2app(W_ArrayInstance.free),
     byptr       = interp2app(W_ArrayInstance.byptr),
     itemaddress = interp2app(W_ArrayInstance.descr_itemaddress),
@@ -223,7 +223,7 @@
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
     __len__     = interp2app(W_ArrayInstance.getlength),
     buffer      = GetSetProperty(W_ArrayInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_ArrayInstance),
+    shape       = interp_attrproperty_w('shape', W_ArrayInstance),
     byptr       = interp2app(W_ArrayInstance.byptr),
     itemaddress = interp2app(W_ArrayInstance.descr_itemaddress),
 )
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -242,7 +242,8 @@
     __new__     = interp2app(descr_new_cdll),
     ptr         = interp2app(W_CDLL.ptr),
     getaddressindll = interp2app(W_CDLL.getaddressindll),
-    name        = interp_attrproperty('name', W_CDLL),
+    name        = interp_attrproperty('name', W_CDLL,
+        wrapfn="newtext_or_none"),
     __doc__     = """ C Dynamically loaded library
 use CDLL(libname) to create a handle to a C library (the argument is processed
 the same way as dlopen processes it). On such a library you can call:
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -3,7 +3,7 @@
 """
 
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import interp_attrproperty
+from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._rawffi.interp_rawffi import segfault_exception, _MS_WINDOWS
@@ -270,8 +270,10 @@
     __call__    = interp2app(W_Structure.descr_call),
     __repr__    = interp2app(W_Structure.descr_repr),
     fromaddress = interp2app(W_Structure.fromaddress),
-    size        = interp_attrproperty('size', W_Structure),
-    alignment   = interp_attrproperty('alignment', W_Structure),
+    size        = interp_attrproperty('size', W_Structure,
+        wrapfn="newint"),
+    alignment   = interp_attrproperty('alignment', W_Structure,
+        wrapfn="newint"),
     fieldoffset = interp2app(W_Structure.descr_fieldoffset),
     fieldsize   = interp2app(W_Structure.descr_fieldsize),
     size_alignment = interp2app(W_Structure.descr_size_alignment),
@@ -382,7 +384,7 @@
     __setattr__ = interp2app(W_StructureInstance.setattr),
     buffer      = GetSetProperty(W_StructureInstance.getbuffer),
     free        = interp2app(W_StructureInstance.free),
-    shape       = interp_attrproperty('shape', W_StructureInstance),
+    shape       = interp_attrproperty_w('shape', W_StructureInstance),
     byptr       = interp2app(W_StructureInstance.byptr),
     fieldaddress= interp2app(W_StructureInstance.descr_fieldaddress),
 )
@@ -403,7 +405,7 @@
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
     buffer      = GetSetProperty(W_StructureInstance.getbuffer),
-    shape       = interp_attrproperty('shape', W_StructureInstance),
+    shape       = interp_attrproperty_w('shape', W_StructureInstance),
     byptr       = interp2app(W_StructureInstance.byptr),
     fieldaddress= interp2app(W_StructureInstance.descr_fieldaddress),
 )
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -406,9 +406,11 @@
     split        = interp2app(W_SRE_Pattern.split_w),
     sub          = interp2app(W_SRE_Pattern.sub_w),
     subn         = interp2app(W_SRE_Pattern.subn_w),
-    flags        = interp_attrproperty('flags', W_SRE_Pattern),
+    flags        = interp_attrproperty('flags', W_SRE_Pattern,
+        wrapfn="newint"),
     groupindex   = interp_attrproperty_w('w_groupindex', W_SRE_Pattern),
-    groups       = interp_attrproperty('num_groups', W_SRE_Pattern),
+    groups       = interp_attrproperty('num_groups', W_SRE_Pattern,
+        wrapfn="newint"),
     pattern      = interp_attrproperty_w('w_pattern', W_SRE_Pattern),
 )
 W_SRE_Pattern.typedef.acceptable_as_base_class = False
@@ -587,7 +589,7 @@
     span         = interp2app(W_SRE_Match.span_w),
     expand       = interp2app(W_SRE_Match.expand_w),
     #
-    re           = interp_attrproperty('srepat', W_SRE_Match),
+    re           = interp_attrproperty_w('srepat', W_SRE_Match),
     string       = GetSetProperty(W_SRE_Match.fget_string),
     pos          = GetSetProperty(W_SRE_Match.fget_pos),
     endpos       = GetSetProperty(W_SRE_Match.fget_endpos),
@@ -649,6 +651,6 @@
     next     = interp2app(W_SRE_Scanner.next_w),
     match    = interp2app(W_SRE_Scanner.match_w),
     search   = interp2app(W_SRE_Scanner.search_w),
-    pattern  = interp_attrproperty('srepat', W_SRE_Scanner),
+    pattern  = interp_attrproperty_w('srepat', W_SRE_Scanner),
 )
 W_SRE_Scanner.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -699,7 +699,8 @@
 W_BZ2Decompressor.typedef = TypeDef("BZ2Decompressor",
     __doc__ = W_BZ2Decompressor.__doc__,
     __new__ = interp2app(descr_decompressor__new__),
-    unused_data = interp_attrproperty("unused_data", W_BZ2Decompressor),
+    unused_data = interp_attrproperty("unused_data", W_BZ2Decompressor,
+        wrapfn="newbytes"),
     decompress = interp2app(W_BZ2Decompressor.decompress),
 )
 
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -249,7 +249,8 @@
     __call__ = interp2app(cfunction_descr_call),
     __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
     __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
-    __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject,
+        wrapfn="newtext_or_none"),
     )
 W_PyCFunctionObject.typedef.acceptable_as_base_class = False
 
@@ -257,7 +258,8 @@
     'method',
     __get__ = interp2app(cmethod_descr_get),
     __call__ = interp2app(cmethod_descr_call),
-    __name__ = interp_attrproperty('name', cls=W_PyCMethodObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCMethodObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCMethodObject),
     __repr__ = interp2app(W_PyCMethodObject.descr_method_repr),
     )
@@ -267,7 +269,8 @@
     'classmethod',
     __get__ = interp2app(cclassmethod_descr_get),
     __call__ = interp2app(cmethod_descr_call),
-    __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject),
+    __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass',
                                          cls=W_PyCClassMethodObject),
     __repr__ = interp2app(W_PyCClassMethodObject.descr_method_repr),
@@ -279,8 +282,10 @@
     'wrapper_descriptor',
     __call__ = interp2app(cwrapper_descr_call),
     __get__ = interp2app(cmethod_descr_get),
-    __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject),
-    __doc__ = interp_attrproperty('doc', cls=W_PyCWrapperObject),
+    __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject,
+        wrapfn="newtext_or_none"),
+    __doc__ = interp_attrproperty('doc', cls=W_PyCWrapperObject,
+        wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCWrapperObject),
     __repr__ = interp2app(W_PyCWrapperObject.descr_method_repr),
     # XXX missing: __getattribute__
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -9,7 +9,7 @@
 from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp_attrproperty, interp2app)
+        interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
@@ -97,9 +97,11 @@
     __get__ = interp2app(GetSetProperty.descr_property_get),
     __set__ = interp2app(GetSetProperty.descr_property_set),
     __delete__ = interp2app(GetSetProperty.descr_property_del),
-    __name__ = interp_attrproperty('name', cls=GetSetProperty),
+    __name__ = interp_attrproperty('name', cls=GetSetProperty,
+        wrapfn="newtext_or_none"),
     __objclass__ = GetSetProperty(GetSetProperty.descr_get_objclass),
-    __doc__ = interp_attrproperty('doc', cls=GetSetProperty),
+    __doc__ = interp_attrproperty('doc', cls=GetSetProperty,
+        wrapfn="newtext_or_none"),
     )
 assert not W_MemberDescr.typedef.acceptable_as_base_class  # no __new__
 
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -1128,12 +1128,12 @@
     __new__ = interp2app(descr__new__),
 
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
-    kind = interp_attrproperty("kind", cls=W_Dtype),
-    char = interp_attrproperty("char", cls=W_Dtype),
-    num = interp_attrproperty("num", cls=W_Dtype),
-    byteorder = interp_attrproperty("byteorder", cls=W_Dtype),
-    itemsize = interp_attrproperty("elsize", cls=W_Dtype),
-    alignment = interp_attrproperty("alignment", cls=W_Dtype),
+    kind = interp_attrproperty("kind", cls=W_Dtype, wrapfn="newtext"),
+    char = interp_attrproperty("char", cls=W_Dtype, wrapfn="newtext"),
+    num = interp_attrproperty("num", cls=W_Dtype, wrapfn="newint"),
+    byteorder = interp_attrproperty("byteorder", cls=W_Dtype, wrapfn="newtext"),
+    itemsize = interp_attrproperty("elsize", cls=W_Dtype, wrapfn="newint"),
+    alignment = interp_attrproperty("alignment", cls=W_Dtype, wrapfn="newint"),
 
     name = GetSetProperty(W_Dtype.descr_get_name),
     str = GetSetProperty(W_Dtype.descr_get_str),
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1199,8 +1199,10 @@
 
     identity = GetSetProperty(W_Ufunc.descr_get_identity),
     accumulate = interp2app(W_Ufunc.descr_accumulate),
-    nin = interp_attrproperty("nin", cls=W_Ufunc),
-    nout = interp_attrproperty("nout", cls=W_Ufunc),
+    nin = interp_attrproperty("nin", cls=W_Ufunc,
+        wrapfn="newint"),
+    nout = interp_attrproperty("nout", cls=W_Ufunc,
+        wrapfn="newint"),
     nargs = interp_attrproperty("nargs", cls=W_Ufunc),
     signature = interp_attrproperty("signature", cls=W_Ufunc),
 
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -199,7 +199,7 @@
     __new__ = interp2app(descr_new_resop),
     __repr__ = interp2app(WrappedOp.descr_repr),
     name = GetSetProperty(WrappedOp.descr_name),
-    offset = interp_attrproperty("offset", cls=WrappedOp),
+    offset = interp_attrproperty("offset", cls=WrappedOp, wrapfn="newint"),
 )
 WrappedOp.typedef.acceptable_as_base_class = False
 
@@ -209,8 +209,8 @@
     __new__ = interp2app(descr_new_guardop),
     __repr__ = interp2app(GuardOp.descr_repr),
     name = GetSetProperty(GuardOp.descr_name),
-    offset = interp_attrproperty("offset", cls=GuardOp),
-    hash = interp_attrproperty("hash", cls=GuardOp),
+    offset = interp_attrproperty("offset", cls=GuardOp, wrapfn="newint"),
+    hash = interp_attrproperty("hash", cls=GuardOp, wrapfn="newint"),
     )
 GuardOp.typedef.acceptable_as_base_class = False
 
@@ -226,9 +226,9 @@
     bytecode_no = GetSetProperty(DebugMergePoint.get_bytecode_no,
                                  doc="offset in the bytecode"),
     call_depth = interp_attrproperty("call_depth", cls=DebugMergePoint,
-                                     doc="Depth of calls within this loop"),
+                                     doc="Depth of calls within this loop", wrapfn="newint"),
     call_id = interp_attrproperty("call_id", cls=DebugMergePoint,
-                     doc="Number of applevel function traced in this loop"),
+                     doc="Number of applevel function traced in this loop", wrapfn="newint"),
     jitdriver_name = GetSetProperty(DebugMergePoint.get_jitdriver_name,
                      doc="Name of the jitdriver 'pypyjit' in the case "
                                     "of the main interpreter loop"),
@@ -311,7 +311,8 @@
     __doc__ = W_JitLoopInfo.__doc__,
     __new__ = interp2app(descr_new_jit_loop_info),
     jitdriver_name = interp_attrproperty('jd_name', cls=W_JitLoopInfo,
-                       doc="Name of the JitDriver, pypyjit for the main one"),
+                       doc="Name of the JitDriver, pypyjit for the main one",
+                       wrapfn="newtext"),
     greenkey = interp_attrproperty_w('w_green_key', cls=W_JitLoopInfo,
                doc="Representation of place where the loop was compiled. "
                     "In the case of the main interpreter loop, it's a triplet "
@@ -319,15 +320,19 @@
     operations = interp_attrproperty_w('w_ops', cls=W_JitLoopInfo, doc=
                                        "List of operations in this loop."),
     loop_no = interp_attrproperty('loop_no', cls=W_JitLoopInfo, doc=
-                                  "Loop cardinal number"),
+                                  "Loop cardinal number",
+                                  wrapfn="newint"),
     bridge_no = GetSetProperty(W_JitLoopInfo.descr_get_bridge_no,
                                doc="bridge number (if a bridge)"),
     type = interp_attrproperty('type', cls=W_JitLoopInfo,
-                               doc="Loop type"),
+                               doc="Loop type",
+                               wrapfn="newtext"),
     asmaddr = interp_attrproperty('asmaddr', cls=W_JitLoopInfo,
-                                  doc="Address of machine code"),
+                                  doc="Address of machine code",
+                                  wrapfn="newint"),
     asmlen = interp_attrproperty('asmlen', cls=W_JitLoopInfo,
-                                  doc="Length of machine code"),
+                                  doc="Length of machine code",
+                                  wrapfn="newint"),
     __repr__ = interp2app(W_JitLoopInfo.descr_repr),
 )
 W_JitLoopInfo.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -134,8 +134,8 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
-    format=interp_attrproperty("format", cls=W_Struct),
-    size=interp_attrproperty("size", cls=W_Struct),
+    format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
+    size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
     pack=interp2app(W_Struct.descr_pack),
     unpack=interp2app(W_Struct.descr_unpack),
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -322,7 +322,8 @@
 
 UCD.typedef = TypeDef("unicodedata.UCD",
                       __doc__ = "",
-                      unidata_version = interp_attrproperty('version', UCD),
+                      unidata_version = interp_attrproperty('version', UCD,
+                          wrapfn="newtext"),
                       **methods)
 
 ucd_3_2_0 = UCD(unicodedb_3_2_0)
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -347,8 +347,8 @@
     __new__ = interp2app(Decompress___new__),
     decompress = interp2app(Decompress.decompress),
     flush = interp2app(Decompress.flush),
-    unused_data = interp_attrproperty('unused_data', Decompress),
-    unconsumed_tail = interp_attrproperty('unconsumed_tail', Decompress),
+    unused_data = interp_attrproperty('unused_data', Decompress, wrapfn="newbytes"),
+    unconsumed_tail = interp_attrproperty('unconsumed_tail', Decompress, wrapfn="newbytes"),
     __doc__ = """decompressobj([wbits]) -- Return a decompressor object.
 
 Optional arg wbits is the window buffer size.

From pypy.commits at gmail.com  Mon Nov 21 05:47:30 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:47:30 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: assign me a new task,
 ass cffi ssl to DONE
Message-ID: <5832d0c2.a3212e0a.fd710.47f0@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5751:5d70acc9f0ae
Date: 2016-11-21 11:47 +0100
http://bitbucket.org/pypy/extradoc/changeset/5d70acc9f0ae/

Log:	assign me a new task, ass cffi ssl to DONE

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,7 +5,7 @@
 In-progress ("Lock" section)
 ----------------------------
 
-(empty)
+* (richard) multiprocessing module changes
 
 
 Misc stuff not formally in any milestone
@@ -88,3 +88,4 @@
 
 * richard: bz2, lzma, ... changes (cpython issue 15955) (DONE)
 * richard: threads do not seem to join (e.g. lib-python/3/test/test_bz2.py) (DONE)
+* richard: cffi ssl (DONE, awaits review + merge)

From pypy.commits at gmail.com  Mon Nov 21 05:54:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:54:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-set-sentinel: close branch
Message-ID: <5832d283.05052e0a.a5b63.431c@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-set-sentinel
Changeset: r88514:9218bb33a6ad
Date: 2016-11-21 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/9218bb33a6ad/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 05:55:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:55:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-bz2-lzma: close branch
Message-ID: <5832d286.d52b190a.c9890.45a2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-bz2-lzma
Changeset: r88515:7fed195ea34f
Date: 2016-11-21 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/7fed195ea34f/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 05:54:57 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:54:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-raffael_t: close branch
Message-ID: <5832d281.4a212e0a.dd805.49d1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-raffael_t
Changeset: r88513:19ec2131220b
Date: 2016-11-21 11:51 +0100
http://bitbucket.org/pypy/pypy/changeset/19ec2131220b/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 05:55:08 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:55:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-async: close branch
Message-ID: <5832d28c.5043190a.885dc.4ba6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-async
Changeset: r88518:7722123bf2e8
Date: 2016-11-21 11:54 +0100
http://bitbucket.org/pypy/pypy/changeset/7722123bf2e8/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 05:55:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:55:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-byteformat: close branch
Message-ID: <5832d288.84c3190a.4cc1b.49ea@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-byteformat
Changeset: r88516:96b8b69e7672
Date: 2016-11-21 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/96b8b69e7672/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 05:55:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 02:55:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-memoryview: close branch
Message-ID: <5832d28a.9d13190a.959ee.46a5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-memoryview
Changeset: r88517:cacb8b75f437
Date: 2016-11-21 11:53 +0100
http://bitbucket.org/pypy/pypy/changeset/cacb8b75f437/

Log:	close branch


From pypy.commits at gmail.com  Mon Nov 21 06:27:00 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 21 Nov 2016 03:27:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: remove part of two if statements that
 always will evaluate to False (cryptography constraint)
Message-ID: <5832da04.50dd190a.8d16f.96af@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88519:0665e64d865e
Date: 2016-11-21 12:26 +0100
http://bitbucket.org/pypy/pypy/changeset/0665e64d865e/

Log:	remove part of two if statements that always will evaluate to False
	(cryptography constraint)

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1133,7 +1133,8 @@
             lib.EC_KEY_free(key)
 
     def set_servername_callback(self, callback):
-        if not HAS_SNI or lib.Cryptography_OPENSSL_NO_TLSEXT:
+        # cryptography constraint: OPENSSL_NO_TLSEXT will never be set!
+        if not HAS_SNI:
             raise NotImplementedError("The TLS extension servername callback, "
                     "SSL_CTX_set_tlsext_servername_callback, "
                     "is not in the current OpenSSL library.")
@@ -1182,7 +1183,8 @@
 
 
 
-if HAS_SNI and not lib.Cryptography_OPENSSL_NO_TLSEXT:
+# cryptography constraint: OPENSSL_NO_TLSEXT will never be set!
+if HAS_SNI:
     @ffi.callback("int(SSL*,int*,void*)")
     def _servername_callback(s, al, arg):
         scb = ffi.from_handle(arg)

From pypy.commits at gmail.com  Mon Nov 21 08:34:27 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 05:34:27 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Add another section,
 put in it a new item
Message-ID: <5832f7e3.9e002e0a.8f1a9.564a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5752:d5ae76521c99
Date: 2016-11-21 14:34 +0100
http://bitbucket.org/pypy/extradoc/changeset/d5ae76521c99/

Log:	Add another section, put in it a new item

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -65,8 +65,8 @@
     sys.settrace(None)
 
 
-Other bugs
-----------
+Non-segfaulting bugs
+--------------------
 
 * on modern Linux: if the first call in the process to
   socketpair() ends in a EINVAL, then cpython will (possibly wrongly)
@@ -83,12 +83,6 @@
 * re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')) -> b'xaz'
   re.sub(b'y', bytearray(b'\\n'), bytearray(b'xyz')) -> internal TypeError
 
-* not a bug: argument clinic turns the "bool" specifier into
-  PyObject_IsTrue(), accepting any argument whatsoever.  This can easily
-  get very confusing for the user, e.g. after messing up the number of
-  arguments.  For example: os.symlink("/path1", "/path2", "/path3")
-  doesn't fail, it just considers the 3rd argument as some true value.
-
 * if you have a stack of generators where each is in 'yield from' from
   the next one, and you call '.next()' on the outermost, then it enters
   and leaves all intermediate frames.  This is costly but may be
@@ -116,11 +110,6 @@
   ``io.UnsupportedOperation.__new__(io.UnsupportedOperation)`` doesn't
   work, but that was not noticed because ``io.UnsupportedOperation()``
   mistakenly works.
- 
-* hash({}.values()) works (but hash({}.keys()) correctly gives
-  TypeError).  That's a bit confusing and, as far as I can tell, always
-  pointless.  Also, related: d.keys()==d.keys() but
-  d.values()!=d.values().
 
 * this program fails the check for no sys.exc_info(), even though at
   the point this assert runs (called from the <== line) we are not in
@@ -179,3 +168,25 @@
                      # which shows that this dict is kept alive by
                      # 'frame'; and we've seen that it is non-empty
                      # as long as we don't read frame.f_locals.
+
+* _collectionsmodule.c: deque_repr uses "[...]" as repr if recursion is
+  detected.  I'd suggest that "deque(...)" is clearer---it's not a list.
+
+
+Other issues of "dubious IMHO" status
+-------------------------------------
+
+* argument clinic turns the "bool" specifier into
+  PyObject_IsTrue(), accepting any argument whatsoever.  This can easily
+  get very confusing for the user, e.g. after messing up the number of
+  arguments.  For example: os.symlink("/path1", "/path2", "/path3")
+  doesn't fail, it just considers the 3rd argument as some true value.
+
+* hash({}.values()) works (but hash({}.keys()) correctly gives
+  TypeError).  That's a bit confusing and, as far as I can tell, always
+  pointless.  Also, related: d.keys()==d.keys() but
+  d.values()!=d.values().
+
+* if you write ``from .a import b`` inside the Python prompt, or in
+  a module not in any package, then you get a SystemError(!) with an
+  error message that is unlikely to help newcomers.

From pypy.commits at gmail.com  Mon Nov 21 08:35:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 05:35:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Tweak test until it passes with -A.
Message-ID: <5832f813.e1002e0a.877ff.5289@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88520:3ef2278cf711
Date: 2016-11-21 12:30 +0100
http://bitbucket.org/pypy/pypy/changeset/3ef2278cf711/

Log:	Tweak test until it passes with -A.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -75,6 +75,9 @@
              abs_sys    = "import sys",
              struct     = "inpackage = 1",
              errno      = "",
+             # Python 3 note: this __future__ has no effect any more,
+             # kept around for testing and to avoid increasing the diff
+             # with PyPy2
              absolute   = "from __future__ import absolute_import\nimport struct",
              relative_b = "from __future__ import absolute_import\nfrom . import struct",
              relative_c = "from __future__ import absolute_import\nfrom .struct import inpackage",
@@ -597,7 +600,7 @@
         imp.reload(sys)
 
         assert sys.path is oldpath
-        assert 'settrace' in dir(sys)
+        assert 'settrace' not in dir(sys)    # at least on CPython 3.5.2
 
     def test_reload_builtin_doesnt_clear(self):
         import imp
@@ -657,16 +660,15 @@
             assert False, 'should not work'
 
     def test_cache_from_source(self):
-        import imp
+        import imp, sys
+        tag = sys.implementation.cache_tag
         pycfile = imp.cache_from_source('a/b/c.py')
-        assert pycfile.startswith('a/b/__pycache__/c.pypy3-')
-        assert pycfile.endswith('.pyc')
-        assert imp.source_from_cache('a/b/__pycache__/c.pypy3-17.pyc'
+        assert pycfile == 'a/b/__pycache__/c.%s.pyc' % tag
+        assert imp.source_from_cache('a/b/__pycache__/c.%s.pyc' % tag
                                      ) == 'a/b/c.py'
         raises(ValueError, imp.source_from_cache, 'a/b/c.py')
 
     def test_invalid_pathname(self):
-        skip("This test fails on CPython 3.3, but passes on CPython 3.4+")
         import imp
         import pkg
         import os
@@ -906,6 +908,10 @@
             stream.close()
 
     def test_pyc_magic_changes(self):
+        # skipped: for now, PyPy generates only one kind of .pyc file
+        # per version.  Different versions should differ in
+        # sys.implementation.cache_tag, which means that they'll look up
+        # different .pyc files anyway.  See test_get_tag() in test_app.py.
         py.test.skip("For now, PyPy generates only one kind of .pyc files")
         # test that the pyc files produced by a space are not reimportable
         # from another, if they differ in what opcodes they support

From pypy.commits at gmail.com  Mon Nov 21 08:38:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 05:38:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: decode the home path in bytes using the
 file system encoding
Message-ID: <5832f8bd.50cc190a.f9d72.50fc@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88521:be4bfc9ac89b
Date: 2016-11-21 14:37 +0100
http://bitbucket.org/pypy/pypy/changeset/be4bfc9ac89b/

Log:	decode the home path in bytes using the file system encoding

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -119,7 +119,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.wrap_fsdecoded(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path

From pypy.commits at gmail.com  Mon Nov 21 09:02:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 06:02:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <5832fe89.8786190a.3ee1b.4e34@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88522:4e312e4af969
Date: 2016-11-21 14:45 +0100
http://bitbucket.org/pypy/pypy/changeset/4e312e4af969/

Log:	Fix test

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1015,7 +1015,9 @@
         sys.meta_path.insert(0, Importer())
         try:
             import math
-            assert len(tried_imports) == 1
+            # the above line may trigger extra imports, like _operator
+            # from app_math.py.  The first one should be 'math'.
+            assert len(tried_imports) >= 1
             package_name = '.'.join(__name__.split('.')[:-1])
             if package_name:
                 assert tried_imports[0][0] == package_name + ".math"

From pypy.commits at gmail.com  Mon Nov 21 09:02:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 06:02:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add an extra hint,
 which makes test_pypy_c pass
Message-ID: <5832fe8b.1a4c2e0a.ca6c7.5356@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88523:88a7cf7c8142
Date: 2016-11-21 15:01 +0100
http://bitbucket.org/pypy/pypy/changeset/88a7cf7c8142/

Log:	Add an extra hint, which makes test_pypy_c pass
	test_exception:test_exception_inside_loop_1.

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -79,9 +79,14 @@
 
     def identifier_w(self, space):
         try:
-            # call the elidable function, with a jit.call_shortcut in case
-            # self._utf8 is already computed
-            identifier = g_identifier_w(self, space)
+            if jit.isconstant(self._value):
+                # constantly encode that unicode string; don't try
+                # to access the cache _utf8 at all
+                identifier = self._value.encode('utf-8')
+            else:
+                # call the elidable function, with a jit.call_shortcut in case
+                # self._utf8 is already computed
+                identifier = g_identifier_w(self, space)
         except UnicodeEncodeError:
             # bah, this is just to get an official app-level
             # UnicodeEncodeError

From pypy.commits at gmail.com  Mon Nov 21 09:02:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 21 Nov 2016 06:02:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <5832fe8d.12092e0a.d908b.54bf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88524:1922742741cc
Date: 2016-11-21 15:02 +0100
http://bitbucket.org/pypy/pypy/changeset/1922742741cc/

Log:	merge heads

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -119,7 +119,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.wrap_fsdecoded(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path

From pypy.commits at gmail.com  Mon Nov 21 09:41:15 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 06:41:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: another case of wrap_fsdecoded
Message-ID: <5833078b.42452e0a.f2fec.565e@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88525:f039aa8f2cce
Date: 2016-11-21 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/f039aa8f2cce/

Log:	another case of wrap_fsdecoded

diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -20,7 +20,7 @@
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
     w_msg = space.wrap(msg)
-    w_filename = space.wrap(fn)
+    w_filename = space.wrap_fsdecoded(fn)
     w_lineno = space.wrap(lineno)
     w_offset = space.wrap(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -30,7 +30,7 @@
                                            'replace')
             w_text = space.wrap(text)
         if self.filename is not None:
-            w_filename = space.fsdecode(space.newbytes(self.filename))
+            w_filename = space.wrap_fsdecoded(self.filename)
         return space.newtuple([space.wrap(self.msg),
                                space.newtuple([w_filename,
                                                space.wrap(self.lineno),

From pypy.commits at gmail.com  Mon Nov 21 09:45:36 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 Nov 2016 06:45:36 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: _pypyjson
Message-ID: <58330890.c54d2e0a.31a89.4b3f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: extradoc
Changeset: r5753:b9a56b065ac2
Date: 2016-11-21 15:45 +0100
http://bitbucket.org/pypy/extradoc/changeset/b9a56b065ac2/

Log:	_pypyjson

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -44,6 +44,9 @@
 * ast compiler: clean up POP_EXCEPT: either remove them, or use it to clean up
   the "finally: name = None; del name" nonsense at the end of any except block
 
+* bonus: use all the features of _pypyjson from the json module again
+  (eg c_encode_basestring_ascii)
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Mon Nov 21 16:09:40 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:40 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: A branch to make the annotation flow
 graph less implicit
Message-ID: <58336294.cd5c190a.4d8f1.7244@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88526:c22676dbe3ed
Date: 2016-11-21 21:08 +0000
http://bitbucket.org/pypy/pypy/changeset/c22676dbe3ed/

Log:	A branch to make the annotation flow graph less implicit


From pypy.commits at gmail.com  Mon Nov 21 16:09:42 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:42 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Simplify code
Message-ID: <58336296.4a212e0a.dd805.71e6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88527:3ade8cb11f89
Date: 2016-11-20 16:57 +0000
http://bitbucket.org/pypy/pypy/changeset/3ade8cb11f89/

Log:	Simplify code

diff --git a/rpython/annotator/dictdef.py b/rpython/annotator/dictdef.py
--- a/rpython/annotator/dictdef.py
+++ b/rpython/annotator/dictdef.py
@@ -1,5 +1,5 @@
 from rpython.annotator.model import (
-    s_ImpossibleValue, SomeInteger, s_Bool, union)
+    s_ImpossibleValue, SomeInteger, s_Bool, union, AnnotatorError)
 from rpython.annotator.listdef import ListItem
 from rpython.rlib.objectmodel import compute_hash
 
@@ -51,23 +51,19 @@
 
         s_key = self.s_value
 
-        def check_eqfn(annotator, graph):
-            s = annotator.binding(graph.getreturnvar())
-            assert s_Bool.contains(s), (
+        s = self.bookkeeper.emulate_pbc_call(
+            myeq, self.s_rdict_eqfn, [s_key, s_key], replace=replace_othereq)
+        if not s_Bool.contains(s):
+            raise AnnotatorError(
                 "the custom eq function of an r_dict must return a boolean"
                 " (got %r)" % (s,))
-        self.bookkeeper.emulate_pbc_call(myeq, self.s_rdict_eqfn, [s_key, s_key],
-                                         replace=replace_othereq,
-                                         callback = check_eqfn)
 
-        def check_hashfn(annotator, graph):
-            s = annotator.binding(graph.getreturnvar())
-            assert SomeInteger().contains(s), (
+        s = self.bookkeeper.emulate_pbc_call(
+            myhash, self.s_rdict_hashfn, [s_key], replace=replace_otherhash)
+        if not SomeInteger().contains(s):
+            raise AnnotatorError(
                 "the custom hash function of an r_dict must return an integer"
                 " (got %r)" % (s,))
-        self.bookkeeper.emulate_pbc_call(myhash, self.s_rdict_hashfn, [s_key],
-                                         replace=replace_otherhash,
-                                         callback = check_hashfn)
 
 
 class DictValue(ListItem):

From pypy.commits at gmail.com  Mon Nov 21 16:09:44 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:44 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Kill confusing function callback case
 in emulate_pbc_call()
Message-ID: <58336298.8da2190a.507c.6a82@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88528:07b0a64492d7
Date: 2016-11-20 17:24 +0000
http://bitbucket.org/pypy/pypy/changeset/07b0a64492d7/

Log:	Kill confusing function callback case in emulate_pbc_call()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -269,17 +269,12 @@
             parent_graph, parent_block, parent_index = whence
             tag = parent_block, parent_index
             self.translator.update_call_graph(parent_graph, graph, tag)
-        # self.notify[graph.returnblock] is a dictionary of call
+        # self.notify[graph.returnblock] is a set of call
         # points to this func which triggers a reflow whenever the
         # return block of this graph has been analysed.
-        callpositions = self.notify.setdefault(graph.returnblock, {})
+        returnpositions = self.notify.setdefault(graph.returnblock, set())
         if whence is not None:
-            if callable(whence):
-                def callback():
-                    whence(self, graph)
-            else:
-                callback = whence
-            callpositions[callback] = True
+            returnpositions.add(whence)
 
         # generalize the function's input arguments
         self.addpendingblock(graph, graph.startblock, inputcells)
@@ -520,12 +515,8 @@
                 self.follow_link(graph, link, constraints)
 
         if block in self.notify:
-            # reflow from certain positions when this block is done
-            for callback in self.notify[block]:
-                if isinstance(callback, tuple):
-                    self.reflowfromposition(callback) # callback is a position
-                else:
-                    callback()
+            for position in self.notify[block]:
+                self.reflowfromposition(position)
 
 
     def follow_link(self, graph, link, constraints):
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -527,10 +527,8 @@
         (position_key, "first") and (position_key, "second").
 
         In general, "unique_key" should somehow uniquely identify where
-        the call is in the source code, and "callback" can be either a
-        position_key to reflow from when we see more general results,
-        or a real callback function that will be called with arguments
-        # "(annotator, called_graph)" whenever the result is generalized.
+        the call is in the source code, and "callback" is a
+        position_key to reflow from when we see more general results.
 
         "replace" can be set to a list of old unique_key values to
         forget now, because the given "unique_key" replaces them.
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -2141,28 +2141,6 @@
         assert (fdesc.get_s_signatures((2, (), False))
                 == [([someint,someint],someint)])
 
-    def test_emulated_pbc_call_callback(self):
-        def f(a,b):
-            return a + b
-        from rpython.annotator import annrpython
-        a = annrpython.RPythonAnnotator()
-        from rpython.annotator import model as annmodel
-
-        memo = []
-        def callb(ann, graph):
-            memo.append(annmodel.SomeInteger() == ann.binding(graph.getreturnvar()))
-
-        s_f = a.bookkeeper.immutablevalue(f)
-        s = a.bookkeeper.emulate_pbc_call('f', s_f, [annmodel.SomeInteger(), annmodel.SomeInteger()],
-                                          callback=callb)
-        assert s == annmodel.SomeImpossibleValue()
-        a.complete()
-
-        assert a.binding(graphof(a, f).getreturnvar()).knowntype == int
-        assert len(memo) >= 1
-        for t in memo:
-            assert t
-
     def test_iterator_union(self):
         def it(d):
             return d.iteritems()

From pypy.commits at gmail.com  Mon Nov 21 16:09:46 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:46 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: small cleanup
Message-ID: <5833629a.521d190a.65478.6acd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88529:93dc170c071f
Date: 2016-11-20 22:11 +0000
http://bitbucket.org/pypy/pypy/changeset/93dc170c071f/

Log:	small cleanup

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -265,15 +265,14 @@
     #___ interface for annotator.bookkeeper _______
 
     def recursivecall(self, graph, whence, inputcells):
-        if isinstance(whence, tuple):
+        if whence is not None:
             parent_graph, parent_block, parent_index = whence
             tag = parent_block, parent_index
             self.translator.update_call_graph(parent_graph, graph, tag)
-        # self.notify[graph.returnblock] is a set of call
-        # points to this func which triggers a reflow whenever the
-        # return block of this graph has been analysed.
-        returnpositions = self.notify.setdefault(graph.returnblock, set())
-        if whence is not None:
+            # self.notify[graph.returnblock] is a set of call
+            # points to this func which triggers a reflow whenever the
+            # return block of this graph has been analysed.
+            returnpositions = self.notify.setdefault(graph.returnblock, set())
             returnpositions.add(whence)
 
         # generalize the function's input arguments

From pypy.commits at gmail.com  Mon Nov 21 16:09:48 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:48 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Create update_var()
Message-ID: <5833629c.1a4b2e0a.d6bae.6b35@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88530:7b89775748cf
Date: 2016-11-21 03:44 +0000
http://bitbucket.org/pypy/pypy/changeset/7b89775748cf/

Log:	Create update_var()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -38,6 +38,7 @@
         self.notify = {}        # {block: {positions-to-reflow-from-when-done}}
         self.fixed_graphs = {}  # set of graphs not to annotate again
         self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
+        self.var_def = {}
         # --- the following information is recorded for debugging ---
         self.blocked_graphs = {} # set of graphs that have blocked blocks
         # --- end of debugging information ---
@@ -264,16 +265,24 @@
 
     #___ interface for annotator.bookkeeper _______
 
+    def record_call(self, graph, whence):
+        parent_graph, parent_block, parent_index = whence
+        tag = parent_block, parent_index
+        self.translator.update_call_graph(parent_graph, graph, tag)
+
+
     def recursivecall(self, graph, whence, inputcells):
         if whence is not None:
-            parent_graph, parent_block, parent_index = whence
-            tag = parent_block, parent_index
-            self.translator.update_call_graph(parent_graph, graph, tag)
+            self.record_call(graph, whence)
+            _, block, index = whence
+            op = block.operations[index]
+            v_result = op.result
+            self.var_def[v_result] = whence
             # self.notify[graph.returnblock] is a set of call
             # points to this func which triggers a reflow whenever the
             # return block of this graph has been analysed.
-            returnpositions = self.notify.setdefault(graph.returnblock, set())
-            returnpositions.add(whence)
+            returnvars = self.notify.setdefault(graph.returnblock, set())
+            returnvars.add(v_result)
 
         # generalize the function's input arguments
         self.addpendingblock(graph, graph.startblock, inputcells)
@@ -287,6 +296,10 @@
             # (some functions actually never do, they always raise exceptions)
             return s_ImpossibleValue
 
+    def update_var(self, v):
+        position_key = self.var_def[v]
+        self.reflowfromposition(position_key)
+
     def reflowfromposition(self, position_key):
         graph, block, index = position_key
         self.reflowpendingblock(graph, block)
@@ -514,9 +527,8 @@
                 self.follow_link(graph, link, constraints)
 
         if block in self.notify:
-            for position in self.notify[block]:
-                self.reflowfromposition(position)
-
+            for v in self.notify[block]:
+                self.update_var(v)
 
     def follow_link(self, graph, link, constraints):
         assert not (isinstance(link.exitcase, (types.ClassType, type)) and

From pypy.commits at gmail.com  Mon Nov 21 16:09:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:54 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Clean up rerased: split interp-level
 ErasingPairIdentity from translator-level IdentityDesc
Message-ID: <583362a2.14012e0a.bbb6e.3e7d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88533:3b26c73f939e
Date: 2016-11-21 19:46 +0000
http://bitbucket.org/pypy/pypy/changeset/3b26c73f939e/

Log:	Clean up rerased: split interp-level ErasingPairIdentity from
	translator-level IdentityDesc

diff --git a/rpython/rlib/rerased.py b/rpython/rlib/rerased.py
--- a/rpython/rlib/rerased.py
+++ b/rpython/rlib/rerased.py
@@ -15,6 +15,8 @@
 """
 
 import sys
+from collections import defaultdict
+
 from rpython.annotator import model as annmodel
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.llannotation import lltype_to_annotation
@@ -48,34 +50,29 @@
     def __deepcopy__(self, memo):
         return self
 
-    def _getdict(self, bk):
-        try:
-            dict = bk._erasing_pairs_tunnel
-        except AttributeError:
-            dict = bk._erasing_pairs_tunnel = {}
-        return dict
+class IdentityDesc(object):
+    def __init__(self, bookkeeper):
+        self.bookkeeper = bookkeeper
+        self.s_input = annmodel.s_ImpossibleValue
+        self.reflowpositions = {}
 
-    def enter_tunnel(self, bookkeeper, s_obj):
-        dict = self._getdict(bookkeeper)
-        s_previousobj, reflowpositions = dict.setdefault(
-            self, (annmodel.s_ImpossibleValue, {}))
-        s_obj = annmodel.unionof(s_previousobj, s_obj)
-        if s_obj != s_previousobj:
-            dict[self] = (s_obj, reflowpositions)
-            for position in reflowpositions:
-                bookkeeper.annotator.reflowfromposition(position)
+    def enter_tunnel(self, s_obj):
+        s_obj = annmodel.unionof(self.s_input, s_obj)
+        if s_obj != self.s_input:
+            self.s_input = s_obj
+            for position in self.reflowpositions:
+                self.bookkeeper.annotator.reflowfromposition(position)
 
-    def leave_tunnel(self, bookkeeper):
-        dict = self._getdict(bookkeeper)
-        s_obj, reflowpositions = dict.setdefault(
-            self, (annmodel.s_ImpossibleValue, {}))
-        reflowpositions[bookkeeper.position_key] = True
-        return s_obj
+    def leave_tunnel(self):
+        self.reflowpositions[self.bookkeeper.position_key] = True
+        return self.s_input
 
-    def get_input_annotation(self, bookkeeper):
-        dict = self._getdict(bookkeeper)
-        s_obj, _ = dict[self]
-        return s_obj
+def _get_desc(bk, identity):
+    try:
+        descs = bk._erasing_pairs_descs
+    except AttributeError:
+        descs = bk._erasing_pairs_descs = defaultdict(lambda: IdentityDesc(bk))
+    return descs[identity]
 
 _identity_for_ints = ErasingPairIdentity("int")
 
@@ -94,21 +91,23 @@
         _about_ = erase
 
         def compute_result_annotation(self, s_obj):
-            identity.enter_tunnel(self.bookkeeper, s_obj)
+            desc = _get_desc(self.bookkeeper, identity)
+            desc.enter_tunnel(s_obj)
             return _some_erased()
 
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
-            s_obj = identity.get_input_annotation(bk)
+            desc = _get_desc(bk, identity)
             hop.exception_cannot_occur()
-            return _rtype_erase(hop, s_obj)
+            return _rtype_erase(hop, desc.s_input)
 
     class Entry(ExtRegistryEntry):
         _about_ = unerase
 
         def compute_result_annotation(self, s_obj):
             assert _some_erased().contains(s_obj)
-            return identity.leave_tunnel(self.bookkeeper)
+            desc = _get_desc(self.bookkeeper, identity)
+            return desc.leave_tunnel()
 
         def specialize_call(self, hop):
             hop.exception_cannot_occur()
@@ -130,6 +129,7 @@
     def __init__(self, x, identity):
         self._x = x
         self._identity = identity
+
     def __repr__(self):
         return "Erased(%r, %r)" % (self._x, self._identity)
 
@@ -140,7 +140,7 @@
             assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(r_self.lowleveltype, value._x * 2 + 1)
         bk = r_self.rtyper.annotator.bookkeeper
-        s_obj = value._identity.get_input_annotation(bk)
+        s_obj = _get_desc(bk, value._identity).s_input
         r_obj = r_self.rtyper.getrepr(s_obj)
         if r_obj.lowleveltype is lltype.Void:
             return lltype.nullptr(r_self.lowleveltype.TO)
@@ -182,9 +182,9 @@
     _type_ = Erased
 
     def compute_annotation(self):
-        identity = self.instance._identity
+        desc = _get_desc(self.bookkeeper, self.instance._identity)
         s_obj = self.bookkeeper.immutablevalue(self.instance._x)
-        identity.enter_tunnel(self.bookkeeper, s_obj)
+        desc.enter_tunnel(s_obj)
         return _some_erased()
 
 # annotation and rtyping support

From pypy.commits at gmail.com  Mon Nov 21 16:09:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:50 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: compute v_result earlier
Message-ID: <5833629e.d7092e0a.c9718.6799@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88531:37d9b461b481
Date: 2016-11-21 05:01 +0000
http://bitbucket.org/pypy/pypy/changeset/37d9b461b481/

Log:	compute v_result earlier

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -270,17 +270,20 @@
         tag = parent_block, parent_index
         self.translator.update_call_graph(parent_graph, graph, tag)
 
+    def get_result_var(self, position_key):
+        _, block, index = position_key
+        op = block.operations[index]
+        v_result = op.result
+        self.var_def[v_result] = position_key
+        return v_result
 
-    def recursivecall(self, graph, whence, inputcells):
+
+    def recursivecall(self, graph, whence, inputcells, v_result):
         if whence is not None:
             self.record_call(graph, whence)
-            _, block, index = whence
-            op = block.operations[index]
-            v_result = op.result
-            self.var_def[v_result] = whence
-            # self.notify[graph.returnblock] is a set of call
-            # points to this func which triggers a reflow whenever the
-            # return block of this graph has been analysed.
+        if v_result is not None:
+            # self.notify[graph.returnblock] is a set of variables to update
+            # whenever the return block of this graph has been analysed.
             returnvars = self.notify.setdefault(graph.returnblock, set())
             returnvars.add(v_result)
 
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -508,9 +508,10 @@
             op = None
             s_previous_result = s_ImpossibleValue
 
+        v_result = self.annotator.get_result_var(whence) if whence is not None else None
         results = []
         for desc in pbc.descriptions:
-            results.append(desc.pycall(whence, args, s_previous_result, op))
+            results.append(desc.pycall(whence, args, s_previous_result, v_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -701,7 +701,7 @@
             self._init_classdef()
         return self.classdef
 
-    def pycall(self, whence, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, v_result, op=None):
         from rpython.annotator.model import SomeInstance, SomeImpossibleValue
         classdef = self.getuniqueclassdef()
         s_instance = SomeInstance(classdef)
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -312,7 +312,7 @@
         else:
             return self.specializer(self, inputcells)
 
-    def pycall(self, whence, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, v_result, op=None):
         inputcells = self.parse_arguments(args)
         result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
@@ -323,7 +323,7 @@
             # recreate the args object because inputcells may have been changed
             new_args = args.unmatch_signature(self.signature, inputcells)
             inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
+            result = annotator.recursivecall(graph, whence, inputcells, v_result)
             signature = getattr(self.pyobj, '_signature_', None)
             if signature:
                 sigresult = enforce_signature_return(self, signature[1], result)
@@ -437,9 +437,9 @@
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
-    def pycall(self, whence, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, v_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(whence, func_args, s_previous_result, op)
+        return self.funcdesc.pycall(whence, func_args, s_previous_result, v_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)
@@ -617,9 +617,9 @@
         s_self = SomePBC([self.frozendesc])
         return args.prepend(s_self)
 
-    def pycall(self, whence, args, s_previous_result, op=None):
+    def pycall(self, whence, args, s_previous_result, v_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(whence, func_args, s_previous_result, op)
+        return self.funcdesc.pycall(whence, func_args, s_previous_result, v_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)

From pypy.commits at gmail.com  Mon Nov 21 16:09:56 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:56 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Use update_var() in rerased
Message-ID: <583362a4.0faa190a.f6d94.6ded@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88534:e25820fb63d8
Date: 2016-11-21 20:58 +0000
http://bitbucket.org/pypy/pypy/changeset/e25820fb63d8/

Log:	Use update_var() in rerased

diff --git a/rpython/rlib/rerased.py b/rpython/rlib/rerased.py
--- a/rpython/rlib/rerased.py
+++ b/rpython/rlib/rerased.py
@@ -40,7 +40,6 @@
 
 
 class ErasingPairIdentity(object):
-
     def __init__(self, name):
         self.name = name
 
@@ -54,17 +53,20 @@
     def __init__(self, bookkeeper):
         self.bookkeeper = bookkeeper
         self.s_input = annmodel.s_ImpossibleValue
-        self.reflowpositions = {}
+        self.updatevars = set()
 
     def enter_tunnel(self, s_obj):
         s_obj = annmodel.unionof(self.s_input, s_obj)
         if s_obj != self.s_input:
             self.s_input = s_obj
-            for position in self.reflowpositions:
-                self.bookkeeper.annotator.reflowfromposition(position)
+            ann = self.bookkeeper.annotator
+            for v in self.updatevars:
+                ann.update_var(v)
 
     def leave_tunnel(self):
-        self.reflowpositions[self.bookkeeper.position_key] = True
+        bk = self.bookkeeper
+        v = bk.annotator.get_result_var(bk.position_key)
+        self.updatevars.add(v)
         return self.s_input
 
 def _get_desc(bk, identity):

From pypy.commits at gmail.com  Mon Nov 21 16:09:52 2016
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 21 Nov 2016 13:09:52 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Remove s_previous_result argument:
Message-ID: <583362a0.99012e0a.c53e5.6b69@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88532:db07794f9b14
Date: 2016-11-21 05:25 +0000
http://bitbucket.org/pypy/pypy/changeset/db07794f9b14/

Log:	Remove s_previous_result argument:

	The annotation that is passed in is, in general, unrelated to the
	thing being annotated.

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -493,25 +493,23 @@
         """
         if emulated is None:
             whence = self.position_key
-            # fish the existing annotation for the result variable,
-            # needed by some kinds of specialization.
-            fn, block, i = self.position_key
+        elif emulated is True:
+            whence = None
+        else:
+            whence = emulated
+
+        if whence is not None:
+            fn, block, i = whence
             op = block.operations[i]
-            s_previous_result = self.annotator.annotation(op.result)
-            if s_previous_result is None:
-                s_previous_result = s_ImpossibleValue
+            v_result = op.result
+            self.annotator.var_def[v_result] = whence
         else:
-            if emulated is True:
-                whence = None
-            else:
-                whence = emulated # callback case
             op = None
-            s_previous_result = s_ImpossibleValue
+            v_result = None
 
-        v_result = self.annotator.get_result_var(whence) if whence is not None else None
         results = []
         for desc in pbc.descriptions:
-            results.append(desc.pycall(whence, args, s_previous_result, v_result, op))
+            results.append(desc.pycall(whence, args, v_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -701,7 +701,7 @@
             self._init_classdef()
         return self.classdef
 
-    def pycall(self, whence, args, s_previous_result, v_result, op=None):
+    def pycall(self, whence, args, v_result, op=None):
         from rpython.annotator.model import SomeInstance, SomeImpossibleValue
         classdef = self.getuniqueclassdef()
         s_instance = SomeInstance(classdef)
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -312,7 +312,7 @@
         else:
             return self.specializer(self, inputcells)
 
-    def pycall(self, whence, args, s_previous_result, v_result, op=None):
+    def pycall(self, whence, args, v_result, op=None):
         inputcells = self.parse_arguments(args)
         result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
@@ -331,11 +331,6 @@
                     annotator.addpendingblock(
                         graph, graph.returnblock, [sigresult])
                     result = sigresult
-        # Some specializations may break the invariant of returning
-        # annotations that are always more general than the previous time.
-        # We restore it here:
-        from rpython.annotator.model import unionof
-        result = unionof(result, s_previous_result)
         return result
 
     def get_graph(self, args, op):
@@ -437,9 +432,9 @@
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
-    def pycall(self, whence, args, s_previous_result, v_result, op=None):
+    def pycall(self, whence, args, v_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(whence, func_args, s_previous_result, v_result, op)
+        return self.funcdesc.pycall(whence, func_args, v_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)
@@ -617,9 +612,9 @@
         s_self = SomePBC([self.frozendesc])
         return args.prepend(s_self)
 
-    def pycall(self, whence, args, s_previous_result, v_result, op=None):
+    def pycall(self, whence, args, v_result, op=None):
         func_args = self.func_args(args)
-        return self.funcdesc.pycall(whence, func_args, s_previous_result, v_result, op)
+        return self.funcdesc.pycall(whence, func_args, v_result, op)
 
     def get_graph(self, args, op):
         func_args = self.func_args(args)

From pypy.commits at gmail.com  Mon Nov 21 19:58:52 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 21 Nov 2016 16:58:52 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix for new use of
 interp_attrproperty
Message-ID: <5833984c.8a29c20a.df6a.45a5@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: space-newtext
Changeset: r88535:09f7c4aa142d
Date: 2016-11-21 15:05 -0800
http://bitbucket.org/pypy/pypy/changeset/09f7c4aa142d/

Log:	fix for new use of interp_attrproperty

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -2,7 +2,7 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty, interp_attrproperty_w
 from pypy.interpreter.baseobjspace import W_Root
 
 from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
@@ -957,7 +957,7 @@
 
 W_CPPClass.typedef = TypeDef(
     'CPPClass',
-    type_name = interp_attrproperty('name', W_CPPClass),
+    type_name = interp_attrproperty('name', W_CPPClass, wrapfn="newtext"),
     get_base_names = interp2app(W_CPPClass.get_base_names),
     get_method_names = interp2app(W_CPPClass.get_method_names),
     get_overload = interp2app(W_CPPClass.get_overload, unwrap_spec=['self', str]),
@@ -984,7 +984,7 @@
 
 W_ComplexCPPClass.typedef = TypeDef(
     'ComplexCPPClass',
-    type_name = interp_attrproperty('name', W_CPPClass),
+    type_name = interp_attrproperty('name', W_CPPClass, wrapfn="newtext"),
     get_base_names = interp2app(W_ComplexCPPClass.get_base_names),
     get_method_names = interp2app(W_ComplexCPPClass.get_method_names),
     get_overload = interp2app(W_ComplexCPPClass.get_overload, unwrap_spec=['self', str]),
@@ -1165,7 +1165,7 @@
 
 W_CPPInstance.typedef = TypeDef(
     'CPPInstance',
-    cppclass = interp_attrproperty('cppclass', cls=W_CPPInstance),
+    cppclass = interp_attrproperty_w('cppclass', cls=W_CPPInstance),
     _python_owns = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns),
     __init__ = interp2app(W_CPPInstance.instance__init__),
     __eq__ = interp2app(W_CPPInstance.instance__eq__),

From pypy.commits at gmail.com  Mon Nov 21 19:58:54 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 21 Nov 2016 16:58:54 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: disable two tests that have
 nothing to do with new_text and are
Message-ID: <5833984e.6a5cc20a.a0487.4daf@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: space-newtext
Changeset: r88536:3bdec0d40070
Date: 2016-11-21 15:49 -0800
http://bitbucket.org/pypy/pypy/changeset/3bdec0d40070/

Log:	disable two tests that have nothing to do with new_text and are in
	fact fixed in cling-support

diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -286,6 +286,8 @@
     def test14_double_enum_trouble(self):
         """Test a redefinition of enum in a derived class"""
 
+        return # don't bother; is fixed in cling-support
+
         import cppyy
 
         M = cppyy.gbl.fragile.M
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -261,12 +261,14 @@
     def test03_string_with_null_character(self):
         """Test that strings with NULL do not get truncated"""
 
+        return # don't bother; is fixed in cling-support
+
         import cppyy
         std = cppyy.gbl.std
         stringy_class = cppyy.gbl.stringy_class
 
         t0 = "aap\0noot"
-        self.assertEqual(t0, "aap\0noot")
+        assert t0 == "aap\0noot"
 
         c, s = stringy_class(""), std.string(t0, len(t0))
 

From pypy.commits at gmail.com  Mon Nov 21 19:58:56 2016
From: pypy.commits at gmail.com (wlav)
Date: Mon, 21 Nov 2016 16:58:56 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: stab at disambiguating
 newint/newlong
Message-ID: <58339850.a285c20a.08c8.4c81@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: space-newtext
Changeset: r88537:be088827c1c4
Date: 2016-11-21 16:57 -0800
http://bitbucket.org/pypy/pypy/changeset/be088827c1c4/

Log:	stab at disambiguating newint/newlong

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -217,6 +217,9 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
+    def _wrap_object(self, space, obj):
+        return getattr(space, self.wrapper)(obj)
+
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
@@ -228,7 +231,7 @@
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
-        return space.wrap(rffiptr[0])
+        return self._wrap_object(space, rffiptr[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = self._get_raw_address(space, w_obj, offset)
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -79,7 +79,7 @@
     _mixin_ = True
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return getattr(space, self.wrapper)(obj)
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
@@ -103,7 +103,7 @@
         self.do_assign = True
 
     def _wrap_object(self, space, obj):
-        return space.wrap(rffi.cast(self.c_type, obj))
+        return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -9,6 +9,17 @@
 # sets of jit_libffi, rffi, and different space unwrapping calls. To get the
 # right mixin, a non-RPython function typeid() is used.
 
+class BaseIntTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['wrapper']
+
+    wrapper     = 'newint'
+
+class BaseLongTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['wrapper']
+
+    wrapper     = 'newlong'
 
 class BoolTypeMixin(object):
     _mixin_     = True
@@ -35,6 +46,7 @@
     libffitype  = jit_libffi.types.schar
     c_type      = rffi.CHAR
     c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.CHARP
+    wrapper     = 'newbytes'
 
     def _unwrap_object(self, space, w_value):
         # allow int to pass to char and make sure that str is of length 1
@@ -52,7 +64,7 @@
                         "char expected, got string of size %d", len(value))
         return value[0] # turn it into a "char" to the annotator
 
-class ShortTypeMixin(object):
+class ShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -63,7 +75,7 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.SHORT, space.int_w(w_obj))
 
-class UShortTypeMixin(object):
+class UShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -74,7 +86,7 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.int_w(w_obj))
 
-class IntTypeMixin(object):
+class IntTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -85,7 +97,7 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.c_int_w(w_obj))
 
-class UIntTypeMixin(object):
+class UIntTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -96,7 +108,7 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.uint_w(w_obj))
 
-class LongTypeMixin(object):
+class LongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -107,7 +119,9 @@
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
-class ULongTypeMixin(object):
+# TODO: check ULong limits; actually, they fail if this is
+#  an BaseLongTypeMixin (i.e. use of space.ewlong) ... why??
+class ULongTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -118,7 +132,7 @@
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
-class LongLongTypeMixin(object):
+class LongLongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -129,7 +143,7 @@
     def _unwrap_object(self, space, w_obj):
         return space.r_longlong_w(w_obj)
 
-class ULongLongTypeMixin(object):
+class ULongLongTypeMixin(BaseLongTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -162,6 +176,7 @@
     libffitype  = jit_libffi.types.double
     c_type      = rffi.DOUBLE
     c_ptrtype   = rffi.DOUBLEP
+    wrapper     = 'newfloat'
     typecode    = 'd'
 
     def _unwrap_object(self, space, w_obj):
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -71,6 +71,10 @@
     typename = "int"
     def __init__(self, val):
         self.val = val
+class FakeLong(FakeBase):
+    typename = "long"
+    def __init__(self, val):
+        self.val = val
 class FakeFloat(FakeBase):
     typename = "float"
     def __init__(self, val):
@@ -145,16 +149,7 @@
     def issequence_w(self, w_obj):
         return True
 
-    @specialize.argtype(1)
     def wrap(self, obj):
-        if isinstance(obj, int):
-            return FakeInt(obj)
-        if isinstance(obj, float):
-            return FakeFloat(obj)
-        if isinstance(obj, str):
-            return FakeString(obj)
-        if isinstance(obj, rffi.r_int):
-            return FakeInt(int(obj))
         assert 0
 
     @specialize.argtype(1)
@@ -163,11 +158,11 @@
 
     @specialize.argtype(1)
     def newint(self, obj):
-        return FakeInt(obj)
+        return FakeInt(rarithmetic.intmask(obj))
 
     @specialize.argtype(1)
     def newlong(self, obj):
-        return FakeInt(obj)
+        return FakeLong(rarithmetic.intmask(obj))
 
     @specialize.argtype(1)
     def newfloat(self, obj):
@@ -207,7 +202,9 @@
         return w_obj.val
 
     def uint_w(self, w_obj):
-        assert isinstance(w_obj, FakeInt)
+        if isinstance(w_obj, FakeInt):
+            return rarithmetic.r_uint(w_obj.val)
+        assert isinstance(w_obj, FakeLong)
         return rarithmetic.r_uint(w_obj.val)
 
     def str_w(self, w_obj):

From pypy.commits at gmail.com  Tue Nov 22 05:46:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 22 Nov 2016 02:46:01 -0800 (PST)
Subject: [pypy-commit] pypy default: Fixes for 8c314acb8dd4
Message-ID: <583421e9.c8111c0a.37ef8.ae40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88538:23ffdb9eb9cc
Date: 2016-11-22 11:44 +0100
http://bitbucket.org/pypy/pypy/changeset/23ffdb9eb9cc/

Log:	Fixes for 8c314acb8dd4

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -495,11 +495,10 @@
             if space.isinstance_w(w_data, space.w_unicode):
                 # note: "encode" is called before we acquire the lock
                 # for this file, which is done in file_write_str().
-                # Direct access to unicodeobject because we don't want
+                # Use a specific space method because we don't want
                 # to call user-defined "encode" methods here.
-                from pypy.objspace.std.unicodeobject import encode_object
-                w_data = encode_object(space, w_data, self.encoding,
-                                       self.errors)
+                w_data = space.encode_unicode_object(w_data,
+                     self.encoding, self.errors)
             data = space.charbuf_w(w_data)
         self.file_write_str(data)
 
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -293,7 +293,7 @@
 same_attributes_as_in_file.remove('__init__')
 same_attributes_as_in_file.extend([
     'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace',
-    'writelines', '__exit__', '__weakref__'])
+    'writelines', '__exit__', '__weakref__', 'write'])
 
 W_BZ2File.typedef = TypeDef(
     "BZ2File",
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -330,6 +330,9 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -626,6 +626,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 

From pypy.commits at gmail.com  Tue Nov 22 05:52:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 22 Nov 2016 02:52:52 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Yet another attempt at
 jit_conditional_call_value
Message-ID: <58342384.4f831c0a.18866.b119@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88539:22fc7a1ddb3d
Date: 2016-11-22 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/22fc7a1ddb3d/

Log:	Yet another attempt at jit_conditional_call_value


From pypy.commits at gmail.com  Tue Nov 22 05:52:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 22 Nov 2016 02:52:54 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: copy and adapt from
 the conditional_call_value_2 branch: goal
Message-ID: <58342386.0f341c0a.bce12.b414@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88540:c73bebe15c32
Date: 2016-11-22 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/c73bebe15c32/

Log:	copy and adapt from the conditional_call_value_2 branch: goal

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1194,10 +1194,19 @@
         return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
-    pass
+    pass           # special-cased below
 
 @specialize.call_location()
 def conditional_call(condition, function, *args):
+    """Does the same as:
+
+         if condition:
+             function(*args)
+
+    but is better for the JIT, in case the condition is often false
+    but could be true occasionally.  It allows the JIT to always produce
+    bridge-free code.  The function is never looked inside.
+    """
     if we_are_jitted():
         _jit_conditional_call(condition, function, *args)
     else:
@@ -1205,21 +1214,58 @@
             function(*args)
 conditional_call._always_inline_ = True
 
+def _jit_conditional_call_value(value, function, *args):
+    return value    # special-cased below
+
+ at specialize.call_location()
+def conditional_call_value(value, function, *args):
+    """Does the same as:
+
+        if not value:
+            value = function(*args)
+        return value
+
+    For the JIT.  Allows one branch which doesn't create a bridge,
+    typically used for caching.  The function must be @elidable.
+    The value and the function's return type must match and cannot
+    be a float.
+    """
+    if we_are_jitted():
+        return _jit_conditional_call_value(value, function, *args)
+    else:
+        if not value:
+            value = function(*args)
+        return value
+conditional_call_value._always_inline_ = True
+
 class ConditionalCallEntry(ExtRegistryEntry):
-    _about_ = _jit_conditional_call
+    _about_ = _jit_conditional_call, _jit_conditional_call_value
 
     def compute_result_annotation(self, *args_s):
-        self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                         args_s[1], args_s[2:])
+        s_res = self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
+                                                 args_s[1], args_s[2:])
+        if self.instance == _jit_conditional_call_value:
+            from rpython.annotator import model as annmodel
+            return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
 
-        args_v = hop.inputargs(lltype.Bool, lltype.Void, *hop.args_r[2:])
+        if self.instance == _jit_conditional_call:
+            opname = 'jit_conditional_call'
+            COND = lltype.Bool
+            resulttype = None
+        elif self.instance == _jit_conditional_call_value:
+            opname = 'jit_conditional_call_value'
+            COND = hop.r_result
+            resulttype = hop.r_result.lowleveltype
+        else:
+            assert False
+        args_v = hop.inputargs(COND, lltype.Void, *hop.args_r[2:])
         args_v[1] = hop.args_r[1].get_concrete_llfn(hop.args_s[1],
                                                     hop.args_s[2:], hop.spaceop)
         hop.exception_is_here()
-        return hop.genop('jit_conditional_call', args_v)
+        return hop.genop(opname, args_v, resulttype=resulttype)
 
 def enter_portal_frame(unique_id):
     """call this when starting to interpret a function. calling this is not
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -3,7 +3,7 @@
 from rpython.conftest import option
 from rpython.annotator.model import UnionError
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
-    JitHintError, oopspec, isconstant, conditional_call,
+    JitHintError, oopspec, isconstant, conditional_call, conditional_call_value,
     elidable, unroll_safe, dont_look_inside,
     enter_portal_frame, leave_portal_frame)
 from rpython.rlib.rarithmetic import r_uint
@@ -302,6 +302,18 @@
         mix.getgraph(later, [annmodel.s_Bool], annmodel.s_None)
         mix.finish()
 
+    def test_conditional_call_value(self):
+        def g(x, y):
+            return x - y + 5
+        def f(n, x, y):
+            return conditional_call_value(n, g, x, y)
+        assert f(0, 1000, 100) == 905
+        res = self.interpret(f, [0, 1000, 100])
+        assert res == 905
+        assert f(-42, 1000, 100) == -42
+        res = self.interpret(f, [-42, 1000, 100])
+        assert res == -42
+
     def test_enter_leave_portal_frame(self):
         from rpython.translator.interactive import Translation
         def g():

From pypy.commits at gmail.com  Tue Nov 22 07:46:17 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:17 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: _winreg,
 and some other space.wrap calls for windows (untested so far)
Message-ID: <58343e19.913fc20a.59c52.36be@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88541:1467bf934968
Date: 2016-11-21 12:51 +0100
http://bitbucket.org/pypy/pypy/changeset/1467bf934968/

Log:	_winreg, and some other space.wrap calls for windows (untested so
	far)

diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -109,7 +109,7 @@
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
 def GetLastError(space):
-    return space.wrap(rwin32.GetLastError_saved())
+    return space.newint(rwin32.GetLastError_saved())
 
 # __________________________________________________________
 # functions for the "win32" namespace
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -617,7 +617,7 @@
 if _MS_WINDOWS:
     @unwrap_spec(code=int)
     def FormatError(space, code):
-        return space.wrap(rwin32.FormatError(code))
+        return space.newtext(rwin32.FormatError(code))
 
     @unwrap_spec(hresult=int)
     def check_HRESULT(space, hresult):
@@ -643,7 +643,7 @@
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_alt_saved())
+        return space.newint(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
         rwin32.SetLastError_alt_saved(error)
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -66,11 +66,11 @@
 
 def w_certEncodingType(space, encodingType):
     if encodingType == X509_ASN_ENCODING:
-        return space.wrap("x509_asn")
+        return space.newtext("x509_asn")
     elif encodingType == PKCS_7_ASN_ENCODING:
-        return space.wrap("pkcs_7_asn")
+        return space.newtext("pkcs_7_asn")
     else:
-        return space.wrap(encodingType)
+        return space.newtext(encodingType)
 
 def w_parseKeyUsage(space, pCertCtx, flags):
     with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -10,8 +10,8 @@
 def raiseWindowsError(space, errcode, context):
     message = rwin32.FormatError(errcode)
     raise OperationError(space.w_WindowsError,
-                         space.newtuple([space.wrap(errcode),
-                                         space.wrap(message)]))
+                         space.newtuple([space.newint(errcode),
+                                         space.newtext(message)]))
 
 class W_HKEY(W_Root):
     def __init__(self, space, hkey):
@@ -26,16 +26,16 @@
         return rffi.cast(rffi.SIZE_T, self.hkey)
 
     def descr_nonzero(self, space):
-        return space.wrap(self.as_int() != 0)
+        return space.newbool(self.as_int() != 0)
 
     def descr_handle_get(self, space):
-        return space.wrap(self.as_int())
+        return space.newint(self.as_int())
 
     def descr_repr(self, space):
-        return space.wrap("<PyHKEY:0x%x>" % (self.as_int(),))
+        return space.newtext("<PyHKEY:0x%x>" % (self.as_int(),))
 
     def descr_int(self, space):
-        return space.wrap(self.as_int())
+        return space.newint(self.as_int())
 
     def descr__enter__(self, space):
         return self
@@ -61,12 +61,12 @@
 On 64 bit windows, the result of this function is a long integer"""
         key = self.as_int()
         self.hkey = rwin32.NULL_HANDLE
-        return space.wrap(key)
+        return space.newint(key)
 
 @unwrap_spec(key=int)
 def new_HKEY(space, w_subtype, key):
     hkey = rffi.cast(rwinreg.HKEY, key)
-    return space.wrap(W_HKEY(space, hkey))
+    return W_HKEY(space, hkey)
 descr_HKEY_new = interp2app(new_HKEY)
 
 W_HKEY.typedef = TypeDef(
@@ -259,7 +259,7 @@
                 if ret != 0:
                     raiseWindowsError(space, ret, 'RegQueryValue')
                 length = intmask(bufsize_p[0] - 1)
-                return space.wrap(rffi.charp2strn(buf, length))
+                return space.newtext(rffi.charp2strn(buf, length))
 
 def convert_to_regdata(space, w_value, typ):
     buf = None
@@ -285,7 +285,7 @@
         else:
             if space.isinstance_w(w_value, space.w_unicode):
                 w_value = space.call_method(w_value, 'encode',
-                                            space.wrap('mbcs'))
+                                            space.newtext('mbcs'))
             buf = rffi.str2charp(space.str_w(w_value))
             buflen = space.len_w(w_value) + 1
 
@@ -305,7 +305,7 @@
                     w_item = space.next(w_iter)
                     if space.isinstance_w(w_item, space.w_unicode):
                         w_item = space.call_method(w_item, 'encode',
-                                                   space.wrap('mbcs'))
+                                                   space.newtext('mbcs'))
                     item = space.str_w(w_item)
                     strings.append(item)
                     buflen += len(item) + 1
@@ -352,9 +352,9 @@
 def convert_from_regdata(space, buf, buflen, typ):
     if typ == rwinreg.REG_DWORD:
         if not buflen:
-            return space.wrap(0)
+            return space.newint(0)
         d = rffi.cast(rwin32.LPDWORD, buf)[0]
-        return space.wrap(d)
+        return space.newint(d)
 
     elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ:
         if not buflen:
@@ -365,8 +365,8 @@
             if buf[buflen - 1] == '\x00':
                 buflen -= 1
             s = rffi.charp2strn(buf, buflen)
-        w_s = space.wrap(s)
-        return space.call_method(w_s, 'decode', space.wrap('mbcs'))
+        w_s = space.newbytes(s)
+        return space.call_method(w_s, 'decode', space.newtext('mbcs'))
 
     elif typ == rwinreg.REG_MULTI_SZ:
         if not buflen:
@@ -381,7 +381,7 @@
             if len(s) == 0:
                 break
             s = ''.join(s)
-            l.append(space.wrap(s))
+            l.append(space.newtext(s))
             i += 1
         return space.newlist(l)
 
@@ -466,7 +466,7 @@
                     return space.newtuple([
                         convert_from_regdata(space, databuf,
                                              length, retType[0]),
-                        space.wrap(intmask(retType[0])),
+                        space.newint(intmask(retType[0])),
                         ])
 
 @unwrap_spec(subkey=str)
@@ -487,7 +487,7 @@
         ret = rwinreg.RegCreateKey(hkey, subkey, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'CreateKey')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(subkey=str, res=int, sam=rffi.r_uint)
 def CreateKeyEx(space, w_hkey, subkey, res=0, sam=rwinreg.KEY_WRITE):
@@ -509,7 +509,7 @@
                                      lltype.nullptr(rwin32.LPDWORD.TO))
         if ret != 0:
             raiseWindowsError(space, ret, 'CreateKeyEx')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(subkey=str)
 def DeleteKey(space, w_hkey, subkey):
@@ -556,7 +556,7 @@
         ret = rwinreg.RegOpenKeyEx(hkey, subkey, res, sam, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'RegOpenKeyEx')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(index=int)
 def EnumValue(space, w_hkey, index):
@@ -615,10 +615,10 @@
 
                             length = intmask(retDataSize[0])
                             return space.newtuple([
-                                space.wrap(rffi.charp2str(valuebuf)),
+                                space.newtext(rffi.charp2str(valuebuf)),
                                 convert_from_regdata(space, databuf,
                                                      length, retType[0]),
-                                space.wrap(intmask(retType[0])),
+                                space.newint(intmask(retType[0])),
                                 ])
 
 @unwrap_spec(index=int)
@@ -648,7 +648,7 @@
                                        lltype.nullptr(rwin32.PFILETIME.TO))
             if ret != 0:
                 raiseWindowsError(space, ret, 'RegEnumKeyEx')
-            return space.wrap(rffi.charp2str(buf))
+            return space.newtext(rffi.charp2str(buf))
 
 def QueryInfoKey(space, w_hkey):
     """tuple = QueryInfoKey(key) - Returns information about a key.
@@ -674,9 +674,9 @@
                     raiseWindowsError(space, ret, 'RegQueryInfoKey')
                 l = ((lltype.r_longlong(ft[0].c_dwHighDateTime) << 32) +
                      lltype.r_longlong(ft[0].c_dwLowDateTime))
-                return space.newtuple([space.wrap(nSubKeys[0]),
-                                       space.wrap(nValues[0]),
-                                       space.wrap(l)])
+                return space.newtuple([space.newint(nSubKeys[0]),
+                                       space.newint(nValues[0]),
+                                       space.newint(l)])
 
 def ConnectRegistry(space, w_machine, w_hkey):
     """key = ConnectRegistry(computer_name, key)
@@ -695,13 +695,13 @@
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
         if ret != 0:
             raiseWindowsError(space, ret, 'RegConnectRegistry')
-        return space.wrap(W_HKEY(space, rethkey[0]))
+        return W_HKEY(space, rethkey[0])
 
 @unwrap_spec(source=unicode)
 def ExpandEnvironmentStrings(space, source):
     "string = ExpandEnvironmentStrings(string) - Expand environment vars."
     try:
-        return space.wrap(rwinreg.ExpandEnvironmentStrings(source))
+        return space.newtext(rwinreg.ExpandEnvironmentStrings(source))
     except WindowsError as e:
         raise wrap_windowserror(space, e)
 

From pypy.commits at gmail.com  Tue Nov 22 07:46:19 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:19 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix test
Message-ID: <58343e1b.42061c0a.85319.6bab@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88542:8e45f3d71f75
Date: 2016-11-21 12:59 +0100
http://bitbucket.org/pypy/pypy/changeset/8e45f3d71f75/

Log:	fix test

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -237,8 +237,10 @@
         class W_C(W_A):
             b = 3
         W_A.typedef = typedef.TypeDef("A",
-            a = typedef.interp_attrproperty("a", cls=W_A),
-            b = typedef.interp_attrproperty("b", cls=W_A),
+            a = typedef.interp_attrproperty("a", cls=W_A,
+                wrapfn="newint"),
+            b = typedef.interp_attrproperty("b", cls=W_A,
+                wrapfn="newint"),
         )
         class W_B(W_Root):
             pass

From pypy.commits at gmail.com  Tue Nov 22 07:46:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:21 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: more windows wraps
Message-ID: <58343e1d.6249c20a.b67cc.3453@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88543:dedd16a4f65a
Date: 2016-11-21 13:04 +0100
http://bitbucket.org/pypy/pypy/changeset/dedd16a4f65a/

Log:	more windows wraps

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -95,7 +95,7 @@
             for i in range(usage.c_cUsageIdentifier):
                 if not usage.c_rgpszUsageIdentifier[i]:
                     continue
-                result_w[i] = space.wrap(rffi.charp2str(
+                result_w[i] = space.newtext(rffi.charp2str(
                     usage.c_rgpszUsageIdentifier[i]))
             return space.newset(result_w)
 
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -220,15 +220,15 @@
     info = rwin32.GetVersionEx()
     w_windows_version_info = app.wget(space, "windows_version_info")
     raw_version = space.newtuple([
-        space.wrap(info[0]),
-        space.wrap(info[1]),
-        space.wrap(info[2]),
-        space.wrap(info[3]),
-        space.wrap(info[4]),
-        space.wrap(info[5]),
-        space.wrap(info[6]),
-        space.wrap(info[7]),
-        space.wrap(info[8]),
+        space.newint(info[0]),
+        space.newint(info[1]),
+        space.newint(info[2]),
+        space.newint(info[3]),
+        space.newtext(info[4]),
+        space.newint(info[5]),
+        space.newint(info[6]),
+        space.newint(info[7]),
+        space.newint(info[8]),
     ])
     return space.call_function(w_windows_version_info, raw_version)
 

From pypy.commits at gmail.com  Tue Nov 22 07:46:22 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:22 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: a number of wraps in cppyy
 (untested)
Message-ID: <58343e1e.96a61c0a.1ea17.dbba@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88544:24290f1e845d
Date: 2016-11-21 13:06 +0100
http://bitbucket.org/pypy/pypy/changeset/24290f1e845d/

Log:	a number of wraps in cppyy (untested)

diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -329,7 +329,7 @@
         cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename))
 
         # cache result
-        space.setattr(w_self, space.newtext('_'+attr), space.wrap(cdata))
+        space.setattr(w_self, space.newtext('_'+attr), cdata)
         return space.getattr(w_self, args_w[0])
 
 class W_TTreeIter(W_Root):
@@ -346,7 +346,7 @@
         space.call_method(w_tree, "SetBranchStatus", space.newtext("*"), space.newint(0))
 
     def iter_w(self):
-        return self.space.wrap(self)
+        return self
 
     def next_w(self):
         if self.current == self.maxentry:
@@ -383,11 +383,11 @@
     ]
 
     for f in allfuncs:
-        _pythonizations[f.__name__] = space.wrap(interp2app(f))
+        _pythonizations[f.__name__] = interp2app(f).spacebind(space)
 
 def _method_alias(space, w_pycppclass, m1, m2):
-    space.setattr(w_pycppclass, space.wrap(m1),
-                  space.getattr(w_pycppclass, space.wrap(m2)))
+    space.setattr(w_pycppclass, space.newtext(m1),
+                  space.getattr(w_pycppclass, space.newtext(m2)))
 
 # callback coming in when app-level bound classes have been created
 def pythonize(space, name, w_pycppclass):
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -60,7 +60,7 @@
         nullarr = arr.fromaddress(space, rffi.cast(rffi.ULONG, 0), 0)
         assert isinstance(nullarr, W_ArrayInstance)
         nullarr.free(space)
-        state.w_nullptr = space.wrap(nullarr)
+        state.w_nullptr = nullarr
     return state.w_nullptr
 
 @unwrap_spec(name=str)
@@ -1069,7 +1069,7 @@
 
     def _get_as_builtin(self):
         try:
-            return self.space.call_method(self.space.wrap(self), "_cppyy_as_builtin")
+            return self.space.call_method(self, "_cppyy_as_builtin")
         except OperationError as e:
             if not (e.match(self.space, self.space.w_TypeError) or
                     e.match(self.space, self.space.w_AttributeError)):

From pypy.commits at gmail.com  Tue Nov 22 07:46:24 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:24 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: some rnadom wraps
Message-ID: <58343e20.6a5cc20a.a0487.3043@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88545:511d80a64542
Date: 2016-11-21 13:07 +0100
http://bitbucket.org/pypy/pypy/changeset/511d80a64542/

Log:	some rnadom wraps

diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -19,4 +19,4 @@
 
     def startup(self, space):
         w_meth_names = fetch_names(space)
-        space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
+        space.setattr(self, space.newtext('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py
--- a/pypy/module/_md5/interp_md5.py
+++ b/pypy/module/_md5/interp_md5.py
@@ -23,7 +23,7 @@
         return self.space.newbytes(self.digest())
 
     def hexdigest_w(self):
-        return self.space.wrap(self.hexdigest())
+        return self.space.newtext(self.hexdigest())
 
     def copy_w(self):
         clone = W_MD5(self.space)
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -39,5 +39,5 @@
         space = self.space
         pypyjitdriver.space = space
         w_obj = space.wrap(PARAMETERS)
-        space.setattr(space.wrap(self), space.wrap('defaults'), w_obj)
+        space.setattr(self, space.newtext('defaults'), w_obj)
         pypy_hooks.space = space
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -300,11 +300,11 @@
                 daylight = int(janzone != julyzone)
                 tzname = [janname, julyname]
 
-    _set_module_object(space, "timezone", space.wrap(timezone))
-    _set_module_object(space, 'daylight', space.wrap(daylight))
-    tzname_w = [space.wrap(tzname[0]), space.wrap(tzname[1])]
+    _set_module_object(space, "timezone", space.newint(timezone))
+    _set_module_object(space, 'daylight', space.newint(daylight))
+    tzname_w = [space.newtext(tzname[0]), space.newtext(tzname[1])]
     _set_module_object(space, 'tzname', space.newtuple(tzname_w))
-    _set_module_object(space, 'altzone', space.wrap(altzone))
+    _set_module_object(space, 'altzone', space.newint(altzone))
 
 def _get_error_msg():
     errno = rposix.get_saved_errno()

From pypy.commits at gmail.com  Tue Nov 22 07:46:30 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:30 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: a number of fixes
Message-ID: <58343e26.4438c20a.48ef1.2cff@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88547:6673bc3db17b
Date: 2016-11-22 13:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6673bc3db17b/

Log:	a number of fixes

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -33,7 +33,7 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.newtext('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.newtext('run_toplevel'))
-        w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
+        w_initstdio = space.getitem(w_dict, space.newtext('initstdio'))
         withjit = space.config.objspace.usemodules.pypyjit
     else:
         w_initstdio = space.appexec([], """():
@@ -112,7 +112,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.newtext(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -20,7 +20,7 @@
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
     w_msg = space.newtext(msg)
-    w_filename = space.wrap(fn)
+    w_filename = space.newtext(fn)
     w_lineno = space.newint(lineno)
     w_offset = space.newint(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -557,6 +557,6 @@
             # when untranslated, we don't wrap into an app-level
             # SystemError (this makes debugging tests harder)
             raise
-        return OperationError(space.w_SystemError, space.wrap(
+        return OperationError(space.w_SystemError, space.newtext(
             "unexpected internal exception (please report a bug): %r%s" %
             (e, extra)))
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -43,8 +43,8 @@
     module = Module(space, w_name)
     if path is not None:
         module.setdictvalue(space, '__file__', space.newtext(path))
-    module.setdictvalue(space, 'ffi', space.newtext(ffi))
-    module.setdictvalue(space, 'lib', space.newtext(lib))
+    module.setdictvalue(space, 'ffi', ffi)
+    module.setdictvalue(space, 'lib', lib)
     w_modules_dict = space.sys.get('modules')
-    space.setitem(w_modules_dict, w_name, space.newtext(module))
-    space.setitem(w_modules_dict, space.newtext(name + '.lib'), space.newtext(lib))
+    space.setitem(w_modules_dict, w_name, module)
+    space.setitem(w_modules_dict, space.newtext(name + '.lib'), lib)
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -617,9 +617,9 @@
                                     "possibly with 'b' or '+' added)",
                                     wrapfn="newtext"),
     encoding = interp_attrproperty('encoding', cls=W_File,
-        wrapfn="wrap_none"),
+        wrapfn="newtext_or_none"),
     errors = interp_attrproperty('errors', cls=W_File,
-        wrapfn="wrap_none"),
+        wrapfn="newtext_or_none"),
     closed   = GetSetProperty(descr_file_closed, cls=W_File,
                               doc="True if the file is closed"),
     newlines = GetSetProperty(descr_file_newlines, cls=W_File,
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -908,7 +908,7 @@
                                              space.bytes_w(w_chunk))
 
             w_decoded = space.call_method(self.w_decoder, "decode",
-                                          w_chunk, space.newbool(cookie.need_eof))
+                                          w_chunk, space.newbool(bool(cookie.need_eof)))
             check_decoded(space, w_decoded)
             self._set_decoded_chars(space.unicode_w(w_decoded))
 
diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py
--- a/pypy/module/_pypyjson/interp_encoder.py
+++ b/pypy/module/_pypyjson/interp_encoder.py
@@ -79,4 +79,4 @@
                 sb.append(HEX[s2 & 0x0f])
 
     res = sb.build()
-    return space.wrap(res)
+    return space.newtext(res)
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -342,7 +342,7 @@
         return space.newint(address_as_uint)
 
     def getidentifier(self, space):
-        return space.wrap(self.cdll.getidentifier())
+        return space.newint(self.cdll.getidentifier())
 
 @unwrap_spec(name='str_or_None', mode=int)
 def descr_new_cdll(space, w_type, name, mode=-1):
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -82,7 +82,7 @@
         retval_w = []
         for fd, revents in retval:
             retval_w.append(space.newtuple([space.newint(fd),
-                                            space.newtext(revents)]))
+                                            space.newint(revents)]))
         return space.newlist(retval_w)
 
 pollmethods = {}
diff --git a/pypy/module/token/__init__.py b/pypy/module/token/__init__.py
--- a/pypy/module/token/__init__.py
+++ b/pypy/module/token/__init__.py
@@ -7,7 +7,7 @@
 
     appleveldefs = {}
     interpleveldefs = {
-        "NT_OFFSET" : "space.wrap(256)",
+        "NT_OFFSET" : "space.newint(256)",
         "ISTERMINAL" : "__init__.isterminal",
         "ISNONTERMINAL" : "__init__.isnonterminal",
         "ISEOF" : "__init__.iseof"
@@ -27,12 +27,12 @@
 
 @unwrap_spec(tok=int)
 def isterminal(space, tok):
-    return space.wrap(tok < 256)
+    return space.newbool(tok < 256)
 
 @unwrap_spec(tok=int)
 def isnonterminal(space, tok):
-    return space.wrap(tok >= 256)
+    return space.newbool(tok >= 256)
 
 @unwrap_spec(tok=int)
 def iseof(space, tok):
-    return space.wrap(tok == pygram.tokens.ENDMARKER)
+    return space.newbool(tok == pygram.tokens.ENDMARKER)
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -2001,6 +2001,7 @@
     _none_value = None
 
     def wrap(self, stringval):
+        assert stringval is not None
         return self.space.newunicode(stringval)
 
     def unwrap(self, w_string):
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -98,7 +98,7 @@
             w_parent_init is not space.w_object):
             # 2.7: warn about excess arguments when both methods are
             # overridden
-            space.warn(space.wrap("object() takes no parameters"),
+            space.warn(space.newtext("object() takes no parameters"),
                        space.w_DeprecationWarning, 1)
         elif (w_parent_new is not space.w_object or
               w_parent_init is space.w_object):
@@ -122,7 +122,7 @@
             w_parent_new is not space.w_object):
             # 2.7: warn about excess arguments when both methods are
             # overridden
-            space.warn(space.wrap("object.__init__() takes no parameters"),
+            space.warn(space.newtext("object.__init__() takes no parameters"),
                        space.w_DeprecationWarning, 1)
         elif (w_parent_init is not space.w_object or
               w_parent_new is space.w_object):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -8,7 +8,7 @@
 from rpython.rlib.objectmodel import instantiate, specialize, is_annotation_constant
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import base_int, widen, is_valid_int
-from rpython.rlib.objectmodel import import_from_mixin
+from rpython.rlib.objectmodel import import_from_mixin, enforceargs
 from rpython.rlib import jit
 
 # Object imports
@@ -340,6 +340,7 @@
         return self.newtext(s)
 
     def newunicode(self, uni):
+        assert uni is not None
         assert isinstance(uni, unicode)
         return W_UnicodeObject(uni)
 
diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py
--- a/pypy/objspace/std/transparent.py
+++ b/pypy/objspace/std/transparent.py
@@ -39,9 +39,9 @@
 def setup(space):
     """Add proxy functions to the __pypy__ module."""
     w___pypy__ = space.getbuiltinmodule("__pypy__")
-    space.setattr(w___pypy__, space.newtext('tproxy'), space.wrap(app_proxy))
+    space.setattr(w___pypy__, space.newtext('tproxy'), app_proxy.spacebind(space))
     space.setattr(w___pypy__, space.newtext('get_tproxy_controller'),
-                  space.wrap(app_proxy_controller))
+                  app_proxy_controller.spacebind(space))
 
 
 def proxy(space, w_type, w_controller):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1,7 +1,8 @@
 """The builtin unicode implementation"""
 
 from rpython.rlib.objectmodel import (
-    compute_hash, compute_unique_id, import_from_mixin)
+    compute_hash, compute_unique_id, import_from_mixin,
+    enforceargs)
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
@@ -29,6 +30,7 @@
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
 
+    @enforceargs(uni=unicode)
     def __init__(self, unistr):
         assert isinstance(unistr, unicode)
         self._value = unistr

From pypy.commits at gmail.com  Tue Nov 22 07:46:29 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 Nov 2016 04:46:29 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: merge default
Message-ID: <58343e25.0209c20a.71be2.3665@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88546:2e449c823765
Date: 2016-11-21 13:17 +0100
http://bitbucket.org/pypy/pypy/changeset/2e449c823765/

Log:	merge default

diff too long, truncating to 2000 out of 30900 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,3 +33,4 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,15 +44,15 @@
   Matti Picus
   Alex Gaynor
   Philip Jenvey
+  Ronan Lamy
   Brian Kearns
-  Ronan Lamy
+  Richard Plangger
   Michael Hudson
   Manuel Jacob
   David Schneider
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Richard Plangger
   Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
@@ -68,8 +68,8 @@
   Niklaus Haldimann
   Camillo Bruni
   Laura Creighton
+  Romain Guillebert
   Toon Verwaest
-  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
   Ronny Pfannschmidt
@@ -89,7 +89,9 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Mark Young
   Alex Martelli
+  Spenser Bauman
   Michal Bendowski
   stian
   Jan de Mooij
@@ -100,20 +102,21 @@
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
+  Stefano Rivera
   Patrick Maupin
+  Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
-  Mark Young
   Timo Paulssen
+  Edd Barrett
   Squeaky
-  Devin Jeanpierre
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
-  Stefano Rivera
   Martin Matusiak
+  Nicolas Truessel
   Konstantin Lopuhin
   Wenzhu Man
   John Witulski
@@ -123,14 +126,12 @@
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Edd Barrett
+  Jeremy Thurgood
   Andreas St&#252;hrk
   Tobias Pape
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Spenser Bauman
-  Jeremy Thurgood
   Pawe&#322; Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -141,7 +142,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Nicolas Truessel
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -156,19 +156,20 @@
   Preston Timmons
   David Ripton
   Jeff Terrace
+  Tim Felgentreff
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
   William Leslie
   Ned Batchelder
-  Tim Felgentreff
   Anton Gulenko
   Amit Regmi
   Ben Young
-  Sergey Matyunin
+  Jasper Schulz
   Nicolas Chauvat
   Andrew Durdin
   Andrew Chambers
+  Sergey Matyunin
   Michael Schneider
   Nicholas Riley
   Jason Chu
@@ -184,16 +185,16 @@
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
-  Sebastian Pawlu&#347;
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  Sebastian Pawlu&#347;
   Stuart Williams
-  Jasper Schulz
-  Christian Hudon
+  Daniel Patrick
+  Aaron Iles
   Toby Watson
   Antoine Pitrou
-  Aaron Iles
+  Christian Hudon
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
@@ -201,8 +202,8 @@
   Stanislaw Halik
   Mikael Sch&#246;nenberg
   Berkin Ilbeyi
+  Faye Zhao
   Elmo M&#228;ntynen
-  Faye Zhao
   Jonathan David Riehl
   Anders Qvist
   Corbin Simpson
@@ -211,11 +212,12 @@
   Alex Perry
   Vaibhav Sood
   Alan McIntyre
+  Reuben Cummings
   Alexander Sedov
   p_zieschang at yahoo.de
   Attila Gobi
-  Jasper.Schulz
   Christopher Pope
+  Aaron Gallagher
   Florin Papa
   Christian Tismer 
   Marc Abramowitz
@@ -232,7 +234,6 @@
   Gabriel
   Lukas Vacek
   Kunal Grover
-  Aaron Gallagher
   Andrew Dalke
   Sylvain Thenault
   Jakub Stasiak
@@ -255,6 +256,7 @@
   Philipp Rustemeuer
   Henrik Vendelbo
   Richard Lancaster
+  Yasir Suhail
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
@@ -267,6 +269,7 @@
   Catalin Gabriel Manciu
   Tomo Cocoa
   Kim Jin Su
+  rafalgalczynski at gmail.com
   Toni Mattis
   Amber Brown
   Lucas Stadler
@@ -294,9 +297,7 @@
   Michael Hudson-Doyle
   Anders Sigfridsson
   Nikolay Zinov
-  Yasir Suhail
   Jason Michalski
-  rafalgalczynski at gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
@@ -310,9 +311,10 @@
   James Lan
   Volodymyr Vladymyrov
   shoma hosaka
-  Daniel Neuh&#228;user
   Ben Mather
   Niclas Olofsson
+  Matthew Miller
+  Rodrigo Ara&#250;jo
   halgari
   Boglarka Vezer
   Chris Pressey
@@ -322,8 +324,9 @@
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Daniil Yarancev
   Jim Baker
-  Rodrigo Ara&#250;jo
+  Dan Crosta
   Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
@@ -337,32 +340,31 @@
   Tomer Chachamu
   Christopher Groskopf
   Asmo Soinio
-  Stefan Marr
   jiaaro
   Mads Kiilerich
-  opassembler.py
   Antony Lee
   Jason Madden
+  Daniel Neuh&#65533;user
+  reubano at gmail.com
   Yaroslav Fedevych
   Jim Hunziker
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
   squeaky
-  Zearin
   soareschen
   Jonas Pfannschmidt
   Kurt Griffiths
   Mike Bayer
-  Matthew Miller
+  Stefan Marr
   Flavio Percoco
   Kristoffer Kleine
-  yasirs
   Michael Chermside
   Anna Ravencroft
   pizi
+  remarkablerocket
   Andrey Churin
-  Dan Crosta
+  Zearin
   Eli Stevens
   Tobias Diaz
   Julien Phalip
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.5.2'
+__version__ = '2.9.2'
diff --git a/_pytest/_argcomplete.py b/_pytest/_argcomplete.py
--- a/_pytest/_argcomplete.py
+++ b/_pytest/_argcomplete.py
@@ -88,9 +88,6 @@
         return completion
 
 if os.environ.get('_ARGCOMPLETE'):
-    # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
-    if sys.version_info[:2] < (2, 6):
-        sys.exit(1)
     try:
         import argcomplete.completers
     except ImportError:
diff --git a/_pytest/_code/__init__.py b/_pytest/_code/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/__init__.py
@@ -0,0 +1,12 @@
+""" python inspection/code generation API """
+from .code import Code  # noqa
+from .code import ExceptionInfo  # noqa
+from .code import Frame  # noqa
+from .code import Traceback  # noqa
+from .code import getrawcode  # noqa
+from .code import patch_builtins  # noqa
+from .code import unpatch_builtins  # noqa
+from .source import Source  # noqa
+from .source import compile_ as compile  # noqa
+from .source import getfslineno  # noqa
+
diff --git a/_pytest/_code/_py2traceback.py b/_pytest/_code/_py2traceback.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/_py2traceback.py
@@ -0,0 +1,81 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/code.py
@@ -0,0 +1,805 @@
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
+
+import py
+
+builtin_repr = repr
+
+reprlib = py.builtin._tryimport('repr', 'reprlib')
+
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from ._py2traceback import format_exception_only
+
+class Code(object):
+    """ wrapper around Python code objects """
+    def __init__(self, rawcode):
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = getrawcode(rawcode)
+        try:
+            self.filename = rawcode.co_filename
+            self.firstlineno = rawcode.co_firstlineno - 1
+            self.name = rawcode.co_name
+        except AttributeError:
+            raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
+
+    def __eq__(self, other):
+        return self.raw == other.raw
+
+    def __ne__(self, other):
+        return not self == other
+
+    @property
+    def path(self):
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
+        p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
+        if not p.check():
+            # XXX maybe try harder like the weird logic
+            # in the standard lib [linecache.updatecache] does?
+            p = self.raw.co_filename
+        return p
+
+    @property
+    def fullsource(self):
+        """ return a _pytest._code.Source object for the full source file of the code
+        """
+        from _pytest._code import source
+        full, _ = source.findsource(self.raw)
+        return full
+
+    def source(self):
+        """ return a _pytest._code.Source object for the code object's source only
+        """
+        # return source only for that part of code
+        import _pytest._code
+        return _pytest._code.Source(self.raw)
+
+    def getargs(self, var=False):
+        """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
+        """
+        # handfull shortcut for getting args
+        raw = self.raw
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
+
+class Frame(object):
+    """Wrapper around a Python frame holding f_locals and f_globals
+    in which expressions can be evaluated."""
+
+    def __init__(self, frame):
+        self.lineno = frame.f_lineno - 1
+        self.f_globals = frame.f_globals
+        self.f_locals = frame.f_locals
+        self.raw = frame
+        self.code = Code(frame.f_code)
+
+    @property
+    def statement(self):
+        """ statement this frame is at """
+        import _pytest._code
+        if self.code.fullsource is None:
+            return _pytest._code.Source("")
+        return self.code.fullsource.getstatement(self.lineno)
+
+    def eval(self, code, **vars):
+        """ evaluate 'code' in the frame
+
+            'vars' are optional additional local variables
+
+            returns the result of the evaluation
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        return eval(code, self.f_globals, f_locals)
+
+    def exec_(self, code, **vars):
+        """ exec 'code' in the frame
+
+            'vars' are optiona; additional local variables
+        """
+        f_locals = self.f_locals.copy()
+        f_locals.update(vars)
+        py.builtin.exec_(code, self.f_globals, f_locals )
+
+    def repr(self, object):
+        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
+        """
+        return py.io.saferepr(object)
+
+    def is_true(self, object):
+        return object
+
+    def getargs(self, var=False):
+        """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
+        """
+        retval = []
+        for arg in self.code.getargs(var):
+            try:
+                retval.append((arg, self.f_locals[arg]))
+            except KeyError:
+                pass     # this can occur when using Psyco
+        return retval
+
+class TracebackEntry(object):
+    """ a single entry in a traceback """
+
+    _repr_style = None
+    exprinfo = None
+
+    def __init__(self, rawentry):
+        self._rawentry = rawentry
+        self.lineno = rawentry.tb_lineno - 1
+
+    def set_repr_style(self, mode):
+        assert mode in ("short", "long")
+        self._repr_style = mode
+
+    @property
+    def frame(self):
+        import _pytest._code
+        return _pytest._code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
+
+    def __repr__(self):
+        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
+
+    @property
+    def statement(self):
+        """ _pytest._code.Source object for the current statement """
+        source = self.frame.code.fullsource
+        return source.getstatement(self.lineno)
+
+    @property
+    def path(self):
+        """ path to the source code """
+        return self.frame.code.path
+
+    def getlocals(self):
+        return self.frame.f_locals
+    locals = property(getlocals, None, None, "locals of underlaying frame")
+
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        from _pytest.assertion.reinterpret import reinterpret
+        if self.exprinfo is None:
+            source = py.builtin._totext(self.statement).strip()
+            x = reinterpret(source, self.frame, should_fail=True)
+            if not py.builtin._istext(x):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
+    def getfirstlinesource(self):
+        # on Jython this firstlineno can be -1 apparently
+        return max(self.frame.code.firstlineno, 0)
+
+    def getsource(self, astcache=None):
+        """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from _pytest._code.source import getstatementrange_ast
+        source = self.frame.code.fullsource
+        if source is None:
+            return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
+        start = self.getfirstlinesource()
+        try:
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
+            end = self.lineno + 1
+        else:
+            if key is not None:
+                astcache[key] = astnode
+        return source[start:end]
+
+    source = property(getsource)
+
+    def ishidden(self):
+        """ return True if the current frame has a var __tracebackhide__
+            resolving to True
+
+            mostly for internal use
+        """
+        try:
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
+
+    def __str__(self):
+        try:
+            fn = str(self.path)
+        except py.error.Error:
+            fn = '???'
+        name = self.frame.code.name
+        try:
+            line = str(self.statement).lstrip()
+        except KeyboardInterrupt:
+            raise
+        except:
+            line = "???"
+        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)
+
+    def name(self):
+        return self.frame.code.raw.co_name
+    name = property(name, None, None, "co_name of underlaying code")
+
+class Traceback(list):
+    """ Traceback objects encapsulate and offer higher level
+        access to Traceback entries.
+    """
+    Entry = TracebackEntry
+    def __init__(self, tb):
+        """ initialize from given python traceback object. """
+        if hasattr(tb, 'tb_next'):
+            def f(cur):
+                while cur is not None:
+                    yield self.Entry(cur)
+                    cur = cur.tb_next
+            list.__init__(self, f(tb))
+        else:
+            list.__init__(self, tb)
+
+    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
+        """ return a Traceback instance wrapping part of this Traceback
+
+            by provding any combination of path, lineno and firstlineno, the
+            first frame to start the to-be-returned traceback is determined
+
+            this allows cutting the first part of a Traceback instance e.g.
+            for formatting reasons (removing some uninteresting bits that deal
+            with handling of the exception/traceback)
+        """
+        for x in self:
+            code = x.frame.code
+            codepath = code.path
+            if ((path is None or codepath == path) and
+                (excludepath is None or not hasattr(codepath, 'relto') or
+                 not codepath.relto(excludepath)) and
+                (lineno is None or x.lineno == lineno) and
+                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
+                return Traceback(x._rawentry)
+        return self
+
+    def __getitem__(self, key):
+        val = super(Traceback, self).__getitem__(key)
+        if isinstance(key, type(slice(0))):
+            val = self.__class__(val)
+        return val
+
+    def filter(self, fn=lambda x: not x.ishidden()):
+        """ return a Traceback instance with certain items removed
+
+            fn is a function that gets a single argument, a TracebackEntry
+            instance, and should return True when the item should be added
+            to the Traceback, False when not
+
+            by default this removes all the TracebackEntries which are hidden
+            (see ishidden() above)
+        """
+        return Traceback(filter(fn, self))
+
+    def getcrashentry(self):
+        """ return last non-hidden traceback entry that lead
+        to the exception of a traceback.
+        """
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
+
+    def recursionindex(self):
+        """ return the index of the frame/TracebackEntry where recursion
+            originates if appropriate, None if no recursion occurred
+        """
+        cache = {}
+        for i, entry in enumerate(self):
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
+            #print "checking for recursion at", key
+            l = cache.setdefault(key, [])
+            if l:
+                f = entry.frame
+                loc = f.f_locals
+                for otherloc in l:
+                    if f.is_true(f.eval(co_equal,
+                        __recursioncache_locals_1=loc,
+                        __recursioncache_locals_2=otherloc)):
+                        return i
+            l.append(entry.frame.f_locals)
+        return None
+
+co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
+                   '?', 'eval')
+
+class ExceptionInfo(object):
+    """ wraps sys.exc_info() objects and offers
+        help for navigating the traceback.
+    """
+    _striptext = ''
+    def __init__(self, tup=None, exprinfo=None):
+        import _pytest._code
+        if tup is None:
+            tup = sys.exc_info()
+            if exprinfo is None and isinstance(tup[1], AssertionError):
+                exprinfo = getattr(tup[1], 'msg', None)
+                if exprinfo is None:
+                    exprinfo = str(tup[1])
+                if exprinfo and exprinfo.startswith('assert '):
+                    self._striptext = 'AssertionError: '
+        self._excinfo = tup
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
+        self.typename = self.type.__name__
+        #: the exception traceback (_pytest._code.Traceback instance)
+        self.traceback = _pytest._code.Traceback(self.tb)
+
+    def __repr__(self):
+        return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
+
+    def exconly(self, tryshort=False):
+        """ return the exception as a string
+
+            when 'tryshort' resolves to True, and the exception is a
+            _pytest._code._AssertionError, only the actual exception part of
+            the exception representation is returned (so 'AssertionError: ' is
+            removed from the beginning)
+        """
+        lines = format_exception_only(self.type, self.value)
+        text = ''.join(lines)
+        text = text.rstrip()
+        if tryshort:
+            if text.startswith(self._striptext):
+                text = text[len(self._striptext):]
+        return text
+
+    def errisinstance(self, exc):
+        """ return True if the exception is an instance of exc """
+        return isinstance(self.value, exc)
+
+    def _getreprcrash(self):
+        exconly = self.exconly(tryshort=True)
+        entry = self.traceback.getcrashentry()
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
+
+    def getrepr(self, showlocals=False, style="long",
+            abspath=False, tbfilter=True, funcargs=False):
+        """ return str()able representation of this exception info.
+            showlocals: show locals per traceback entry
+            style: long|short|no|native traceback style
+            tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
+        """
+        if style == 'native':
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
+
+        fmt = FormattedExcinfo(showlocals=showlocals, style=style,
+            abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
+        return fmt.repr_excinfo(self)
+
+    def __str__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return str(loc)
+
+    def __unicode__(self):
+        entry = self.traceback[-1]
+        loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())
+        return unicode(loc)
+
+
+class FormattedExcinfo(object):
+    """ presenting information about failing Functions and Generators. """
+    # for traceback entries
+    flow_marker = ">"
+    fail_marker = "E"
+
+    def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False):
+        self.showlocals = showlocals
+        self.style = style
+        self.tbfilter = tbfilter
+        self.funcargs = funcargs
+        self.abspath = abspath
+        self.astcache = {}
+
+    def _getindent(self, source):
+        # figure out indent for given source
+        try:
+            s = str(source.getstatement(len(source)-1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            try:
+                s = str(source[-1])
+            except KeyboardInterrupt:
+                raise
+            except:
+                return 0
+        return 4 + (len(s) - len(s.lstrip()))
+
+    def _getentrysource(self, entry):
+        source = entry.getsource(self.astcache)
+        if source is not None:
+            source = source.deindent()
+        return source
+
+    def _saferepr(self, obj):
+        return py.io.saferepr(obj)
+
+    def repr_args(self, entry):
+        if self.funcargs:
+            args = []
+            for argname, argvalue in entry.frame.getargs(var=True):
+                args.append((argname, self._saferepr(argvalue)))
+            return ReprFuncArgs(args)
+
+    def get_source(self, source, line_index=-1, excinfo=None, short=False):
+        """ return formatted and marked up source lines. """
+        import _pytest._code
+        lines = []
+        if source is None or line_index >= len(source.lines):
+            source = _pytest._code.Source("???")
+            line_index = 0
+        if line_index < 0:
+            line_index += len(source)
+        space_prefix = "    "
+        if short:
+            lines.append(space_prefix + source.lines[line_index].strip())
+        else:
+            for line in source.lines[:line_index]:
+                lines.append(space_prefix + line)
+            lines.append(self.flow_marker + "   " + source.lines[line_index])
+            for line in source.lines[line_index+1:]:
+                lines.append(space_prefix + line)
+        if excinfo is not None:
+            indent = 4 if short else self._getindent(source)
+            lines.extend(self.get_exconly(excinfo, indent=indent, markall=True))
+        return lines
+
+    def get_exconly(self, excinfo, indent=4, markall=False):
+        lines = []
+        indent = " " * indent
+        # get the real exception information out
+        exlines = excinfo.exconly(tryshort=True).split('\n')
+        failindent = self.fail_marker + indent[1:]
+        for line in exlines:
+            lines.append(failindent + line)
+            if not markall:
+                failindent = indent
+        return lines
+
+    def repr_locals(self, locals):
+        if self.showlocals:
+            lines = []
+            keys = [loc for loc in locals if loc[0] != "@"]
+            keys.sort()
+            for name in keys:
+                value = locals[name]
+                if name == '__builtins__':
+                    lines.append("__builtins__ = <builtins>")
+                else:
+                    # This formatting could all be handled by the
+                    # _repr() function, which is only reprlib.Repr in
+                    # disguise, so is very configurable.
+                    str_repr = self._saferepr(value)
+                    #if len(str_repr) < 70 or not isinstance(value,
+                    #                            (list, tuple, dict)):
+                    lines.append("%-10s = %s" %(name, str_repr))
+                    #else:
+                    #    self._line("%-10s =\\" % (name,))
+                    #    # XXX
+                    #    py.std.pprint.pprint(value, stream=self.excinfowriter)
+            return ReprLocals(lines)
+
+    def repr_traceback_entry(self, entry, excinfo=None):
+        import _pytest._code
+        source = self._getentrysource(entry)
+        if source is None:
+            source = _pytest._code.Source("???")
+            line_index = 0
+        else:
+            # entry.getfirstlinesource() can be -1, should be 0 on jython
+            line_index = entry.lineno - max(entry.getfirstlinesource(), 0)
+
+        lines = []
+        style = entry._repr_style
+        if style is None:
+            style = self.style
+        if style in ("short", "long"):
+            short = style == "short"
+            reprargs = self.repr_args(entry) if not short else None
+            s = self.get_source(source, line_index, excinfo, short=short)
+            lines.extend(s)
+            if short:
+                message = "in %s" %(entry.name)
+            else:
+                message = excinfo and excinfo.typename or ""
+            path = self._makepath(entry.path)
+            filelocrepr = ReprFileLocation(path, entry.lineno+1, message)
+            localsrepr = None
+            if not short:
+                localsrepr =  self.repr_locals(entry.locals)
+            return ReprEntry(lines, reprargs, localsrepr, filelocrepr, style)
+        if excinfo:
+            lines.extend(self.get_exconly(excinfo, indent=4))
+        return ReprEntry(lines, None, None, None, style)
+
+    def _makepath(self, path):
+        if not self.abspath:
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
+            if len(np) < len(str(path)):
+                path = np
+        return path
+
+    def repr_traceback(self, excinfo):
+        traceback = excinfo.traceback
+        if self.tbfilter:
+            traceback = traceback.filter()
+        recursionindex = None
+        if is_recursion_error(excinfo):
+            recursionindex = traceback.recursionindex()
+        last = traceback[-1]
+        entries = []
+        extraline = None
+        for index, entry in enumerate(traceback):
+            einfo = (last == entry) and excinfo or None
+            reprentry = self.repr_traceback_entry(entry, einfo)
+            entries.append(reprentry)
+            if index == recursionindex:
+                extraline = "!!! Recursion detected (same locals & position)"
+                break
+        return ReprTraceback(entries, extraline, style=self.style)
+
+    def repr_excinfo(self, excinfo):
+        reprtraceback = self.repr_traceback(excinfo)
+        reprcrash = excinfo._getreprcrash()
+        return ReprExceptionInfo(reprtraceback, reprcrash)
+
+class TerminalRepr:
+    def __str__(self):
+        s = self.__unicode__()
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return s
+
+    def __unicode__(self):
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
+        self.toterminal(tw)
+        return io.getvalue().strip()
+
+    def __repr__(self):
+        return "<%s instance at %0x>" %(self.__class__, id(self))
+
+
+class ReprExceptionInfo(TerminalRepr):
+    def __init__(self, reprtraceback, reprcrash):
+        self.reprtraceback = reprtraceback
+        self.reprcrash = reprcrash
+        self.sections = []
+
+    def addsection(self, name, content, sep="-"):
+        self.sections.append((name, content, sep))
+
+    def toterminal(self, tw):
+        self.reprtraceback.toterminal(tw)
+        for name, content, sep in self.sections:
+            tw.sep(sep, name)
+            tw.line(content)
+
+class ReprTraceback(TerminalRepr):
+    entrysep = "_ "
+
+    def __init__(self, reprentries, extraline, style):
+        self.reprentries = reprentries
+        self.extraline = extraline
+        self.style = style
+
+    def toterminal(self, tw):
+        # the entries might have different styles
+        for i, entry in enumerate(self.reprentries):
+            if entry.style == "long":
+                tw.line("")
+            entry.toterminal(tw)
+            if i < len(self.reprentries) - 1:
+                next_entry = self.reprentries[i+1]
+                if entry.style == "long" or \
+                   entry.style == "short" and next_entry.style == "long":
+                    tw.sep(self.entrysep)
+
+        if self.extraline:
+            tw.line(self.extraline)
+
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    style = "native"
+
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
+class ReprEntry(TerminalRepr):
+    localssep = "_ "
+
+    def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr, style):
+        self.lines = lines
+        self.reprfuncargs = reprfuncargs
+        self.reprlocals = reprlocals
+        self.reprfileloc = filelocrepr
+        self.style = style
+
+    def toterminal(self, tw):
+        if self.style == "short":
+            self.reprfileloc.toterminal(tw)
+            for line in self.lines:
+                red = line.startswith("E   ")
+                tw.line(line, bold=True, red=red)
+            #tw.line("")
+            return
+        if self.reprfuncargs:
+            self.reprfuncargs.toterminal(tw)
+        for line in self.lines:
+            red = line.startswith("E   ")
+            tw.line(line, bold=True, red=red)
+        if self.reprlocals:
+            #tw.sep(self.localssep, "Locals")
+            tw.line("")
+            self.reprlocals.toterminal(tw)
+        if self.reprfileloc:
+            if self.lines:
+                tw.line("")
+            self.reprfileloc.toterminal(tw)
+
+    def __str__(self):
+        return "%s\n%s\n%s" % ("\n".join(self.lines),
+                               self.reprlocals,
+                               self.reprfileloc)
+
+class ReprFileLocation(TerminalRepr):
+    def __init__(self, path, lineno, message):
+        self.path = str(path)
+        self.lineno = lineno
+        self.message = message
+
+    def toterminal(self, tw):
+        # filename and lineno output for each entry,
+        # using an output format that most editors unterstand
+        msg = self.message
+        i = msg.find("\n")
+        if i != -1:
+            msg = msg[:i]
+        tw.line("%s:%s: %s" %(self.path, self.lineno, msg))
+
+class ReprLocals(TerminalRepr):
+    def __init__(self, lines):
+        self.lines = lines
+
+    def toterminal(self, tw):
+        for line in self.lines:
+            tw.line(line)
+
+class ReprFuncArgs(TerminalRepr):
+    def __init__(self, args):
+        self.args = args
+
+    def toterminal(self, tw):
+        if self.args:
+            linesofar = ""
+            for name, value in self.args:
+                ns = "%s = %s" %(name, value)
+                if len(ns) + len(linesofar) + 2 > tw.fullwidth:
+                    if linesofar:
+                        tw.line(linesofar)
+                    linesofar =  ns
+                else:
+                    if linesofar:
+                        linesofar += ", " + ns
+                    else:
+                        linesofar = ns
+            if linesofar:
+                tw.line(linesofar)
+            tw.line("")
+
+
+
+oldbuiltins = {}
+
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from _pytest.assertion import reinterpret
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = reinterpret.AssertionError
+    if compile:
+        import _pytest._code
+        l = oldbuiltins.setdefault('compile', [])
+        l.append(py.builtin.builtins.compile)
+        py.builtin.builtins.compile = _pytest._code.compile
+
+def unpatch_builtins(assertion=True, compile=True):
+    """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
+    if compile:
+        py.builtin.builtins.compile = oldbuiltins['compile'].pop()
+
+def getrawcode(obj, trycall=True):
+    """ return code object for given function. """
+    try:
+        return obj.__code__
+    except AttributeError:
+        obj = getattr(obj, 'im_func', obj)
+        obj = getattr(obj, 'func_code', obj)
+        obj = getattr(obj, 'f_code', obj)
+        obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
+        return obj
+
+if sys.version_info[:2] >= (3, 5):  # RecursionError introduced in 3.5
+    def is_recursion_error(excinfo):
+        return excinfo.errisinstance(RecursionError)  # noqa
+else:
+    def is_recursion_error(excinfo):
+        if not excinfo.errisinstance(RuntimeError):
+            return False
+        try:
+            return "maximum recursion depth exceeded" in str(excinfo.value)
+        except UnicodeError:
+            return False
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_code/source.py
@@ -0,0 +1,423 @@
+from __future__ import generators
+
+from bisect import bisect_right
+import sys
+import inspect, tokenize
+import py
+from types import ModuleType
+cpy_compile = compile
+
+try:
+    import _ast
+    from _ast import PyCF_ONLY_AST as _AST_FLAG
+except ImportError:
+    _AST_FLAG = 0
+    _ast = None
+
+
+class Source(object):
+    """ a immutable object holding a source code fragment,
+        possibly deindenting it.
+    """
+    _compilecounter = 0
+    def __init__(self, *parts, **kwargs):
+        self.lines = lines = []
+        de = kwargs.get('deindent', True)
+        rstrip = kwargs.get('rstrip', True)
+        for part in parts:
+            if not part:
+                partlines = []
+            if isinstance(part, Source):
+                partlines = part.lines
+            elif isinstance(part, (tuple, list)):
+                partlines = [x.rstrip("\n") for x in part]
+            elif isinstance(part, py.builtin._basestring):
+                partlines = part.split('\n')
+                if rstrip:
+                    while partlines:
+                        if partlines[-1].strip():
+                            break
+                        partlines.pop()
+            else:
+                partlines = getsource(part, deindent=de).lines
+            if de:
+                partlines = deindent(partlines)
+            lines.extend(partlines)
+
+    def __eq__(self, other):
+        try:
+            return self.lines == other.lines
+        except AttributeError:
+            if isinstance(other, str):
+                return str(self) == other
+            return False
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.lines[key]
+        else:
+            if key.step not in (None, 1):
+                raise IndexError("cannot slice a Source with a step")
+            return self.__getslice__(key.start, key.stop)
+
+    def __len__(self):
+        return len(self.lines)
+
+    def __getslice__(self, start, end):
+        newsource = Source()
+        newsource.lines = self.lines[start:end]
+        return newsource
+
+    def strip(self):
+        """ return new source object with trailing
+            and leading blank lines removed.
+        """
+        start, end = 0, len(self)
+        while start < end and not self.lines[start].strip():
+            start += 1
+        while end > start and not self.lines[end-1].strip():
+            end -= 1
+        source = Source()
+        source.lines[:] = self.lines[start:end]
+        return source
+
+    def putaround(self, before='', after='', indent=' ' * 4):
+        """ return a copy of the source object with
+            'before' and 'after' wrapped around it.
+        """
+        before = Source(before)
+        after = Source(after)
+        newsource = Source()
+        lines = [ (indent + line) for line in self.lines]
+        newsource.lines = before.lines + lines +  after.lines
+        return newsource
+
+    def indent(self, indent=' ' * 4):
+        """ return a copy of the source object with
+            all lines indented by the given indent-string.
+        """
+        newsource = Source()
+        newsource.lines = [(indent+line) for line in self.lines]
+        return newsource
+
+    def getstatement(self, lineno, assertion=False):
+        """ return Source statement which contains the
+            given linenumber (counted from 0).
+        """
+        start, end = self.getstatementrange(lineno, assertion)
+        return self[start:end]
+
+    def getstatementrange(self, lineno, assertion=False):
+        """ return (start, end) tuple which spans the minimal
+            statement region which containing the given lineno.
+        """
+        if not (0 <= lineno < len(self)):
+            raise IndexError("lineno out of range")
+        ast, start, end = getstatementrange_ast(lineno, self)
+        return start, end
+
+    def deindent(self, offset=None):
+        """ return a new source object deindented by offset.
+            If offset is None then guess an indentation offset from
+            the first non-blank line.  Subsequent lines which have a
+            lower indentation offset will be copied verbatim as
+            they are assumed to be part of multilines.
+        """
+        # XXX maybe use the tokenizer to properly handle multiline
+        #     strings etc.pp?
+        newsource = Source()
+        newsource.lines[:] = deindent(self.lines, offset)
+        return newsource
+
+    def isparseable(self, deindent=True):
+        """ return True if source is parseable, heuristically
+            deindenting it by default.
+        """
+        try:
+            import parser
+        except ImportError:
+            syntax_checker = lambda x: compile(x, 'asd', 'exec')
+        else:
+            syntax_checker = parser.suite
+
+        if deindent:
+            source = str(self.deindent())
+        else:
+            source = str(self)
+        try:
+            #compile(source+'\n', "x", "exec")
+            syntax_checker(source+'\n')
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return False
+        else:
+            return True
+
+    def __str__(self):
+        return "\n".join(self.lines)
+
+    def compile(self, filename=None, mode='exec',
+                flag=generators.compiler_flag,
+                dont_inherit=0, _genframe=None):
+        """ return compiled code object. if filename is None
+            invent an artificial filename which displays
+            the source/line position of the caller frame.
+        """
+        if not filename or py.path.local(filename).check(file=0):
+            if _genframe is None:
+                _genframe = sys._getframe(1) # the caller
+            fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno
+            base = "<%d-codegen " % self._compilecounter
+            self.__class__._compilecounter += 1
+            if not filename:
+                filename = base + '%s:%d>' % (fn, lineno)
+            else:
+                filename = base + '%r %s:%d>' % (filename, fn, lineno)
+        source = "\n".join(self.lines) + '\n'
+        try:
+            co = cpy_compile(source, filename, mode, flag)
+        except SyntaxError:
+            ex = sys.exc_info()[1]
+            # re-represent syntax errors from parsing python strings
+            msglines = self.lines[:ex.lineno]
+            if ex.offset:
+                msglines.append(" "*ex.offset + '^')
+            msglines.append("(code was compiled probably from here: %s)" % filename)
+            newex = SyntaxError('\n'.join(msglines))
+            newex.offset = ex.offset
+            newex.lineno = ex.lineno
+            newex.text = ex.text
+            raise newex
+        else:
+            if flag & _AST_FLAG:
+                return co
+            lines = [(x + "\n") for x in self.lines]
+            if sys.version_info[0] >= 3:
+                # XXX py3's inspect.getsourcefile() checks for a module
+                # and a pep302 __loader__ ... we don't have a module
+                # at code compile-time so we need to fake it here
+                m = ModuleType("_pycodecompile_pseudo_module")
+                py.std.inspect.modulesbyfile[filename] = None
+                py.std.sys.modules[None] = m
+                m.__loader__ = 1
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
+            return co
+
+#
+# public API shortcut functions
+#
+
+def compile_(source, filename=None, mode='exec', flags=
+            generators.compiler_flag, dont_inherit=0):
+    """ compile the given source to a raw code object,
+        and maintain an internal cache which allows later
+        retrieval of the source code for the code object
+        and any recursively created code objects.
+    """
+    if _ast is not None and isinstance(source, _ast.AST):
+        # XXX should Source support having AST?
+        return cpy_compile(source, filename, mode, flags, dont_inherit)
+    _genframe = sys._getframe(1) # the caller
+    s = Source(source)
+    co = s.compile(filename, mode, flags, _genframe=_genframe)
+    return co
+
+
+def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
+    import _pytest._code
+    try:
+        code = _pytest._code.Code(obj)
+    except TypeError:
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
+        fspath = fn and py.path.local(fn) or None
+        lineno = -1
+        if fspath:
+            try:
+                _, lineno = findsource(obj)
+            except IOError:
+                pass
+    else:
+        fspath = code.path
+        lineno = code.firstlineno
+    assert isinstance(lineno, int)
+    return fspath, lineno
+
+#
+# helper functions
+#
+
+def findsource(obj):
+    try:
+        sourcelines, lineno = py.std.inspect.findsource(obj)
+    except py.builtin._sysex:
+        raise
+    except:
+        return None, -1
+    source = Source()
+    source.lines = [line.rstrip() for line in sourcelines]
+    return source, lineno
+
+def getsource(obj, **kwargs):
+    import _pytest._code
+    obj = _pytest._code.getrawcode(obj)
+    try:
+        strsrc = inspect.getsource(obj)
+    except IndentationError:
+        strsrc = "\"Buggy python version consider upgrading, cannot get source\""
+    assert isinstance(strsrc, str)
+    return Source(strsrc, **kwargs)
+
+def deindent(lines, offset=None):
+    if offset is None:
+        for line in lines:
+            line = line.expandtabs()
+            s = line.lstrip()
+            if s:
+                offset = len(line)-len(s)
+                break
+        else:
+            offset = 0
+    if offset == 0:
+        return list(lines)
+    newlines = []
+    def readline_generator(lines):
+        for line in lines:
+            yield line + '\n'
+        while True:
+            yield ''
+
+    it = readline_generator(lines)
+
+    try:
+        for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(lambda: next(it)):
+            if sline > len(lines):
+                break # End of input reached
+            if sline > len(newlines):
+                line = lines[sline - 1].expandtabs()
+                if line.lstrip() and line[:offset].isspace():
+                    line = line[offset:] # Deindent
+                newlines.append(line)
+
+            for i in range(sline, eline):
+                # Don't deindent continuing lines of
+                # multiline tokens (i.e. multiline strings)
+                newlines.append(lines[i])
+    except (IndentationError, tokenize.TokenError):
+        pass
+    # Add any lines we didn't see. E.g. if an exception was raised.
+    newlines.extend(lines[len(newlines):])
+    return newlines
+
+
+def get_statement_startend2(lineno, node):
+    import ast
+    # flatten all statements and except handlers into one lineno-list
+    # AST's line numbers start indexing at 1
+    l = []
+    for x in ast.walk(node):
+        if isinstance(x, _ast.stmt) or isinstance(x, _ast.ExceptHandler):
+            l.append(x.lineno - 1)
+            for name in "finalbody", "orelse":
+                val = getattr(x, name, None)
+                if val:
+                    # treat the finally/orelse part as its own statement
+                    l.append(val[0].lineno - 1 - 1)
+    l.sort()
+    insert_index = bisect_right(l, lineno)
+    start = l[insert_index - 1]
+    if insert_index >= len(l):
+        end = None
+    else:
+        end = l[insert_index]
+    return start, end
+
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend2(lineno, astnode)
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - there might be empty lines
+    # - we might have lesser indented code blocks at the end
+    if end is None:
+        end = len(source.lines)
+
+    if end > start + 1:
+        # make sure we don't span differently indented code blocks
+        # by using the BlockFinder helper used which inspect.getsource() uses itself
+        block_finder = inspect.BlockFinder()
+        # if we start with an indented line, put blockfinder to "started" mode
+        block_finder.started = source.lines[start][0].isspace()
+        it = ((x + "\n") for x in source.lines[start:end])
+        try:
+            for tok in tokenize.generate_tokens(lambda: next(it)):
+                block_finder.tokeneater(*tok)
+        except (inspect.EndOfBlock, IndentationError):
+            end = block_finder.last + start
+        except Exception:
+            pass
+
+    # the end might still point to a comment or empty line, correct it
+    while end:
+        line = source.lines[end - 1].lstrip()
+        if line.startswith("#") or not line:
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/_pytest/_pluggy.py b/_pytest/_pluggy.py
new file mode 100644
--- /dev/null
+++ b/_pytest/_pluggy.py
@@ -0,0 +1,11 @@
+"""
+imports symbols from vendored "pluggy" if available, otherwise
+falls back to importing "pluggy" from the default namespace.
+"""
+
+try:
+    from _pytest.vendored_packages.pluggy import *  # noqa
+    from _pytest.vendored_packages.pluggy import __version__  # noqa
+except ImportError:
+    from pluggy import *  # noqa
+    from pluggy import __version__  # noqa
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,24 +2,37 @@
 support for presenting detailed information in failing assertions.
 """
 import py
+import os
 import sys
 from _pytest.monkeypatch import monkeypatch
 from _pytest.assertion import util
 
+
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
+    group.addoption('--assert',
+                    action="store",
+                    dest="assertmode",
                     choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', '--no-magic', action="store_true",
-        default=False, help="DEPRECATED equivalent to --assert=plain")
+                    default="rewrite",
+                    metavar="MODE",
+                    help="""control assertion debugging tools.  'plain'
+                            performs no assertion debugging.  'reinterp'
+                            reinterprets assert statements after they failed
+                            to provide assertion expression information.
+                            'rewrite' (the default) rewrites assert
+                            statements in test modules on import to
+                            provide assert expression information. """)
+    group.addoption('--no-assert',
+                    action="store_true",
+                    default=False,
+                    dest="noassert",
+                    help="DEPRECATED equivalent to --assert=plain")
+    group.addoption('--nomagic', '--no-magic',
+                    action="store_true",
+                    default=False,
+                    help="DEPRECATED equivalent to --assert=plain")
+
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -28,6 +41,7 @@
         self.mode = mode
         self.trace = config.trace.root.get("assertion")
 
+
 def pytest_configure(config):
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
@@ -41,7 +55,7 @@
             # Both Jython and CPython 2.6.0 have AST bugs that make the
             # assertion rewriting hook malfunction.
             if (sys.platform.startswith('java') or
-                sys.version_info[:3] == (2, 6, 0)):
+                    sys.version_info[:3] == (2, 6, 0)):
                 mode = "reinterp"
     if mode != "plain":
         _load_modules(mode)
@@ -57,11 +71,12 @@
     config._assertstate = AssertionState(config, mode)
     config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
+    def undo():
+        hook = config._assertstate.hook
+        if hook is not None and hook in sys.meta_path:
+            sys.meta_path.remove(hook)
+    config.add_cleanup(undo)
 
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
 
 def pytest_collection(session):
     # this hook is only called when test modules are collected
@@ -71,36 +86,66 @@
     if hook is not None:
         hook.set_session(session)
 
+
+def _running_on_ci():
+    """Check if we're currently running on a CI system."""
+    env_vars = ['CI', 'BUILD_NUMBER']
+    return any(var in os.environ for var in env_vars)
+
+
 def pytest_runtest_setup(item):
+    """Setup the pytest_assertrepr_compare hook
+
+    The newinterpret and rewrite modules will use util._reprcompare if
+    it exists to use custom reporting via the
+    pytest_assertrepr_compare hook.  This sets up this custom
+    comparison for the test.
+    """
     def callbinrepr(op, left, right):
+        """Call the pytest_assertrepr_compare hook and prepare the result
+
+        This uses the first result from the hook and then ensures the
+        following:
+        * Overly verbose explanations are dropped unless -vv was used or
+          running on a CI.
+        * Embedded newlines are escaped to help util.format_explanation()
+          later.
+        * If the rewrite mode is used embedded %-characters are replaced
+          to protect later % formatting.
+
+        The result can be formatted by util.format_explanation() for
+        pretty printing.
+        """
         hook_result = item.ihook.pytest_assertrepr_compare(
             config=item.config, op=op, left=left, right=right)
-
         for new_expl in hook_result:
             if new_expl:
-                # Don't include pageloads of data unless we are very
-                # verbose (-vv)
-                if (sum(len(p) for p in new_expl[1:]) > 80*8
-                        and item.config.option.verbose < 2):
-                    new_expl[1:] = [py.builtin._totext(
-                        'Detailed information truncated, use "-vv" to show')]
-                res = py.builtin._totext('\n~').join(new_expl)
+                if (sum(len(p) for p in new_expl[1:]) > 80*8 and
+                        item.config.option.verbose < 2 and
+                        not _running_on_ci()):
+                    show_max = 10
+                    truncated_lines = len(new_expl) - show_max
+                    new_expl[show_max:] = [py.builtin._totext(
+                        'Detailed information truncated (%d more lines)'
+                        ', use "-vv" to show' % truncated_lines)]
+                new_expl = [line.replace("\n", "\\n") for line in new_expl]
+                res = py.builtin._totext("\n~").join(new_expl)
                 if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
                     res = res.replace("%", "%%")
                 return res
     util._reprcompare = callbinrepr
 
+
 def pytest_runtest_teardown(item):
     util._reprcompare = None
 
+
 def pytest_sessionfinish(session):
     hook = session.config._assertstate.hook
     if hook is not None:
         hook.session = None
 
+
 def _load_modules(mode):
     """Lazily import assertion related code."""
     global rewrite, reinterpret
@@ -108,6 +153,7 @@
     if mode == "rewrite":
         from _pytest.assertion import rewrite  # noqa
 
+
 def warn_about_missing_assertion(mode):
     try:
         assert False
@@ -121,8 +167,10 @@
             specifically = "failing tests may report as passing"
 
         sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
+                         " because assert statements are not executed "
+                         "by the underlying Python interpreter "
+                         "(are you using python -O?)\n")
 
+
+# Expose this plugin's implementation for the pytest_assertrepr_compare hook
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java"):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,

From pypy.commits at gmail.com  Tue Nov 22 11:44:17 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 08:44:17 -0800 (PST)
Subject: [pypy-commit] pypy default: A failing test showing a pervasive
 issue with unions of objects of different types
Message-ID: <583475e1.46831c0a.41ab4.4076@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88548:b0e65fd580b2
Date: 2016-11-22 16:30 +0000
http://bitbucket.org/pypy/pypy/changeset/b0e65fd580b2/

Log:	A failing test showing a pervasive issue with unions of objects of
	different types

diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)

From pypy.commits at gmail.com  Tue Nov 22 14:02:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 11:02:57 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: Hack around to make the flag
 do something (disallowing int-float unions)
Message-ID: <58349661.6737c20a.58d18.ccbf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88550:98f1bef80edd
Date: 2016-11-22 18:44 +0000
http://bitbucket.org/pypy/pypy/changeset/98f1bef80edd/

Log:	Hack around to make the flag do something (disallowing int-float
	unions)

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -50,7 +50,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
-        self.allow_bad_unions = True  # temporary feature flag, see config.translation.brokentypes
+        # temporary feature flag, see config.translation.brokentypes
+        # defaults to True in real translations
+        self.allow_bad_unions = False
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -76,9 +78,7 @@
         # make input arguments and set their type
         args_s = [self.typeannotation(t) for t in input_arg_types]
 
-        # XXX hack
-        annmodel.TLS.check_str_without_nul = (
-            self.translator.config.translation.check_str_without_nul)
+        self._setup_union_hacks()  # XXX
 
         flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
 
@@ -86,6 +86,11 @@
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
+    def _setup_union_hacks(self):
+        annmodel.TLS.check_str_without_nul = (
+            self.translator.config.translation.check_str_without_nul)
+        annmodel.TLS.allow_bad_unions = self.allow_bad_unions
+
     def get_call_parameters(self, function, args_s, policy):
         desc = self.bookkeeper.getdesc(function)
         prevpolicy = self.policy
@@ -101,9 +106,7 @@
         if policy is None:
             from rpython.annotator.policy import AnnotatorPolicy
             policy = AnnotatorPolicy()
-            # XXX hack
-            annmodel.TLS.check_str_without_nul = (
-                self.translator.config.translation.check_str_without_nul)
+            self._setup_union_hacks()  # XXX
         graph, inputcells = self.get_call_parameters(function, args_s, policy)
         self.build_graph_types(graph, inputcells, complete_now=False)
         self.complete_helpers(policy)
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -436,14 +436,16 @@
 class __extend__(pairtype(SomeFloat, SomeFloat)):
 
     def union((flt1, flt2)):
-        if not TLS.allow_int_to_float:
+        if not TLS.allow_bad_unions or not TLS.allow_int_to_float:
             # in this mode, if one of the two is actually the
             # subclass SomeInteger, complain
             if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
                 raise UnionError(flt1, flt2)
         return SomeFloat()
 
-    add = sub = mul = union
+    def add((flt1, flt2)):
+        return SomeFloat()
+    sub = mul = add
 
     def div((flt1, flt2)):
         return SomeFloat()
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -45,6 +45,7 @@
     # the no_nul attribute...
     check_str_without_nul = False
     allow_int_to_float = True
+    allow_bad_unions = False
 TLS = State()
 
 class SomeObject(object):
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -14,7 +14,10 @@
 @pytest.fixture()
 def annotator():
     t = TranslationContext()
-    return t.buildannotator()
+    t.config.translation.brokentypes = False
+    ann = t.buildannotator()
+    ann._setup_union_hacks()
+    return ann
 
 
 listdef1 = ListDef(None, SomeTuple([SomeInteger(nonneg=True), SomeString()]))

From pypy.commits at gmail.com  Tue Nov 22 14:02:53 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 11:02:53 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: Add translation flag (no
 effect yet)
Message-ID: <5834965d.54b31c0a.e216c.7d03@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88549:2520706e5bb0
Date: 2016-11-22 17:36 +0000
http://bitbucket.org/pypy/pypy/changeset/2520706e5bb0/

Log:	Add translation flag (no effect yet)

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -50,6 +50,7 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.allow_bad_unions = True  # temporary feature flag, see config.translation.brokentypes
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -181,6 +181,11 @@
                "attempt to pre-allocate the list",
                default=False,
                cmdline='--listcompr'),
+    BoolOption("brokentypes",
+               "Allow unsafe type unions in the annotator for "
+               "backwards compatibility.",
+               default=True,
+               cmdline='--broken-types'),
     IntOption("withsmallfuncsets",
               "Represent groups of less funtions than this as indices into an array",
                default=0),
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -68,6 +68,7 @@
             raise ValueError("we already have an annotator")
         from rpython.annotator.annrpython import RPythonAnnotator
         self.annotator = RPythonAnnotator(self, policy=policy)
+        self.annotator.allow_bad_unions = self.config.translation.brokentypes
         return self.annotator
 
     def buildrtyper(self):

From pypy.commits at gmail.com  Tue Nov 22 14:07:31 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 11:07:31 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: just change the default config
 option to see what breaks
Message-ID: <58349773.4673c20a.e67f3.cf66@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88551:1d3afc4858e4
Date: 2016-11-22 19:06 +0000
http://bitbucket.org/pypy/pypy/changeset/1d3afc4858e4/

Log:	just change the default config option to see what breaks

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -184,7 +184,7 @@
     BoolOption("brokentypes",
                "Allow unsafe type unions in the annotator for "
                "backwards compatibility.",
-               default=True,
+               default=False,
                cmdline='--broken-types'),
     IntOption("withsmallfuncsets",
               "Represent groups of less funtions than this as indices into an array",

From pypy.commits at gmail.com  Tue Nov 22 15:33:09 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Nov 2016 12:33:09 -0800 (PST)
Subject: [pypy-commit] pypy default: test, fix for PyObject_Format('hello',
 u'') returning unicode (go figure)
Message-ID: <5834ab85.4673c20a.e67f3.eec7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88552:299c1e591591
Date: 2016-11-22 22:12 +0200
http://bitbucket.org/pypy/pypy/changeset/299c1e591591/

Log:	test, fix for PyObject_Format('hello', u'') returning unicode (go
	figure)

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -252,7 +252,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.wrap('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_Unicode(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -312,6 +312,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """

From pypy.commits at gmail.com  Tue Nov 22 15:33:12 2016
From: pypy.commits at gmail.com (mattip)
Date: Tue, 22 Nov 2016 12:33:12 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <5834ab88.a285c20a.08c8.f0b8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88553:1bdb2e50a3ba
Date: 2016-11-22 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/1bdb2e50a3ba/

Log:	merge default into branch

diff --git a/_pytest/_code/code.py b/_pytest/_code/code.py
--- a/_pytest/_code/code.py
+++ b/_pytest/_code/code.py
@@ -601,7 +601,7 @@
 class TerminalRepr:
     def __str__(self):
         s = self.__unicode__()
-        if sys.version_info[0] < 3:
+        if sys.version_info[0] < 3 and isinstance(s, unicode):
             s = s.encode('utf-8')
         return s
 
diff --git a/_pytest/_code/source.py b/_pytest/_code/source.py
--- a/_pytest/_code/source.py
+++ b/_pytest/_code/source.py
@@ -416,6 +416,8 @@
             trysource = source[start:end]
             if trysource.isparseable():
                 return start, end
+            if end == start + 100:   # XXX otherwise, it takes forever
+                break                # XXX
     raise SyntaxError("no valid source range around line %d " % (lineno,))
 
 
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -302,24 +302,7 @@
             size = space.r_longlong_w(w_size)
         stream.truncate(size)
 
-    def direct_write(self, w_data):
-        space = self.space
-        self.check_writable()
-        if self.binary:
-            data = space.getarg_w('s*', w_data).as_str()
-        else:
-            if space.isinstance_w(w_data, space.w_unicode):
-                w_errors = w_encoding = None
-                if self.encoding:
-                    w_encoding = space.wrap(self.encoding)
-                if self.errors:
-                    w_errors = space.wrap(self.errors)
-                w_data = space.call_method(w_data, "encode",
-                                           w_encoding, w_errors)
-            data = space.charbuf_w(w_data)
-        self.do_direct_write(data)
-
-    def do_direct_write(self, data):
+    def direct_write_str(self, data):
         self.softspace = 0
         self.getstream().write(data)
 
@@ -349,7 +332,7 @@
     _exposed_method_names = []
 
     def _decl(class_scope, name, docstring,
-              wrapresult="space.wrap(result)"):
+              wrapresult="space.wrap(result)", exposed=True):
         # hack hack to build a wrapper around the direct_xxx methods.
         # The wrapper adds lock/unlock calls and a space.wrap() on
         # the result, conversion of stream errors to OperationErrors,
@@ -389,7 +372,8 @@
         exec str(src) in globals(), class_scope
         if unwrap_spec is not None:
             class_scope['file_' + name].unwrap_spec = unwrap_spec
-        class_scope['_exposed_method_names'].append(name)
+        if exposed:
+            class_scope['_exposed_method_names'].append(name)
 
 
     _decl(locals(), "__init__", """Opens a file.""")
@@ -466,11 +450,8 @@
 
 Size defaults to the current file position, as returned by tell().""")
 
-    _decl(locals(), "write",
-        """write(str) -> None.  Write string str to file.
-
-Note that due to buffering, flush() or close() may be needed before
-the file on disk reflects the data written.""")
+    _decl(locals(), "write_str", "Interp-level only, see file_write()",
+          exposed=False)
 
     _decl(locals(), "__iter__",
         """Iterating over files, as in 'for line in f:', returns each line of
@@ -501,6 +482,26 @@
         else:
             return space.str_w(space.repr(w_name))
 
+    def file_write(self, w_data):
+        """write(str) -> None.  Write string str to file.
+
+Note that due to buffering, flush() or close() may be needed before
+the file on disk reflects the data written."""
+        space = self.space
+        self.check_writable()
+        if self.binary:
+            data = space.getarg_w('s*', w_data).as_str()
+        else:
+            if space.isinstance_w(w_data, space.w_unicode):
+                # note: "encode" is called before we acquire the lock
+                # for this file, which is done in file_write_str().
+                # Use a specific space method because we don't want
+                # to call user-defined "encode" methods here.
+                w_data = space.encode_unicode_object(w_data,
+                     self.encoding, self.errors)
+            data = space.charbuf_w(w_data)
+        self.file_write_str(data)
+
     def file_writelines(self, w_lines):
         """writelines(sequence_of_strings) -> None.  Write the strings to the file.
 
@@ -612,6 +613,7 @@
                               cls=W_File,
                               doc="Support for 'print'."),
     __repr__ = interp2app(W_File.file__repr__),
+    write      = interp2app(W_File.file_write),
     writelines = interp2app(W_File.file_writelines),
     __exit__ = interp2app(W_File.file__exit__),
     __weakref__ = make_weakref_descr(W_File),
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -293,7 +293,7 @@
 same_attributes_as_in_file.remove('__init__')
 same_attributes_as_in_file.extend([
     'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace',
-    'writelines', '__exit__', '__weakref__'])
+    'writelines', '__exit__', '__weakref__', 'write'])
 
 W_BZ2File.typedef = TypeDef(
     "BZ2File",
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -252,7 +252,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.wrap('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_Unicode(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -350,7 +350,6 @@
                 PyObject * obj;
                 char * data;
                 int len;
-                PyType_Ready(&PyStringArrType_Type);
 
                 data = PyString_AS_STRING(args);
                 len = PyString_GET_SIZE(args);
@@ -434,6 +433,7 @@
                 PyStringArrType_Type.tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE;
                 PyStringArrType_Type.tp_itemsize = sizeof(char);
                 PyStringArrType_Type.tp_base = &PyString_Type;
+                if (PyType_Ready(&PyStringArrType_Type) < 0) INITERROR;
             ''')
 
         a = module.newsubstr('abc')
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -139,13 +139,6 @@
             return result;
             ''')
         assert 'foo\0bar\0baz' == pybuffer(buffer('foo\0bar\0baz'))
-        import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            class A(object):
-                def __buffer__(self, flags):
-                    return buffer('123')
-            assert pybuffer(A()) == '123'
-
 
     def test_pyarg_parse_string_fails(self):
         """
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py
--- a/pypy/module/cpyext/test/test_intobject.py
+++ b/pypy/module/cpyext/test/test_intobject.py
@@ -85,7 +85,6 @@
                 if (!PyArg_ParseTuple(args, "Oi", &name, &intval))
                     return NULL;
 
-                PyType_Ready(&Enum_Type);
                 enumObj = PyObject_New(EnumObject, &Enum_Type);
                 if (!enumObj) {
                     return NULL;
@@ -160,7 +159,8 @@
                 /*tp_new*/              0
             };
             """, more_init = '''
-            Enum_Type.tp_base = &PyInt_Type;
+                Enum_Type.tp_base = &PyInt_Type;
+                if (PyType_Ready(&Enum_Type) < 0) INITERROR;
             ''')
 
         a = module.newEnum("ULTIMATE_ANSWER", 42)
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -29,11 +29,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_length = mp_length;
-                tp_as_mapping.mp_subscript = mp_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
@@ -44,8 +39,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue = '''
             static PyObject *
             mp_subscript(PyObject *self, PyObject *key)
             {
@@ -61,6 +55,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_length = mp_length;
+                tp_as_mapping.mp_subscript = mp_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj["hi there"] == 42
@@ -80,11 +80,6 @@
            ("test", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_length = sq_length;
-                tp_as_sequence.sq_item = sq_item;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''),
@@ -94,8 +89,7 @@
                     PySequence_Check(args) +
                     PyMapping_Check(args) * 2);
             ''')
-            ],
-            '''
+            ], prologue='''
             static PyObject *
             sq_item(PyObject *self, Py_ssize_t size)
             {
@@ -111,6 +105,12 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_length = sq_length;
+                tp_as_sequence.sq_item = sq_item;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.test()
         assert obj[1] == 42
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -233,9 +233,9 @@
         if cls.runappdirect:
             try:
                 import numpy
-                cls.w_numpy_include = [numpy.get_include()]
-            except:
+            except ImportError:
                 skip('numpy not importable')
+            cls.w_numpy_include = [numpy.get_include()]
         else:
             numpy_incl = os.path.abspath(os.path.dirname(__file__) +
                                          '/../include/_numpypy')
@@ -306,9 +306,6 @@
                 ),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifdef PYPY_VERSION
@@ -351,9 +348,6 @@
                 '''),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 ''',
@@ -409,9 +403,6 @@
                 """),
                 ], include_dirs=self.numpy_include,
                    prologue='''
-                #ifdef PYPY_VERSION
-                    #include <pypy_numpy.h>
-                #endif
                 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
                 #include <numpy/arrayobject.h>
                 #ifndef PYPY_VERSION
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -312,6 +312,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -713,15 +713,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_mapping = &tp_as_mapping;
-                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
@@ -736,6 +731,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init = '''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_mapping = &tp_as_mapping;
+                tp_as_mapping.mp_ass_subscript = mp_ass_subscript;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
@@ -747,15 +747,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_as_sequence = &tp_as_sequence;
-                tp_as_sequence.sq_contains = sq_contains;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static int
             sq_contains(PyObject *self, PyObject *value)
             {
@@ -766,6 +761,11 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         obj = module.new_obj()
         res = "foo" in obj
@@ -906,11 +906,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type.tp_as_number = &intlike_as_number;
-                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
-                intlike_as_number.nb_add = intlike_nb_add;
-                intlike_as_number.nb_power = intlike_nb_pow;
-                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
                 intObj = PyObject_New(IntLikeObject, &IntLike_Type);
                 if (!intObj) {
                     return NULL;
@@ -927,8 +922,6 @@
                 if (!PyArg_ParseTuple(args, "l", &intval))
                     return NULL;
 
-                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
-                if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL;
                 intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp);
                 if (!intObjNoOp) {
                     return NULL;
@@ -936,8 +929,7 @@
 
                 intObjNoOp->ival = intval;
                 return (PyObject *)intObjNoOp;
-             """)], prologue=
-            """
+             """)], prologue="""
             #include <math.h>
             typedef struct
             {
@@ -989,6 +981,14 @@
                 /*tp_name*/             "IntLikeNoOp",
                 /*tp_basicsize*/        sizeof(IntLikeObjectNoOp),
             };
+            """, more_init="""
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                IntLike_Type.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
+                intlike_as_number.nb_add = intlike_nb_add;
+                intlike_as_number.nb_power = intlike_nb_pow;
+                if (PyType_Ready(&IntLike_Type) < 0) INITERROR;
+                IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES;
+                if (PyType_Ready(&IntLike_Type_NoOp) < 0) INITERROR;
             """)
         a = module.newInt(1)
         b = module.newInt(2)
@@ -1039,19 +1039,11 @@
         else:
             raise AssertionError("did not get TypeError!")
 
-    def test_call_tp_dealloc_when_created_from_python(self):
+    def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
             ("fetchFooType", "METH_VARARGS",
              """
                 PyObject *o;
-                Foo_Type.tp_basicsize = sizeof(FooObject);
-                Foo_Type.tp_dealloc = &dealloc_foo;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES
-                                    | Py_TPFLAGS_BASETYPE;
-                Foo_Type.tp_new = &new_foo;
-                Foo_Type.tp_free = &PyObject_Del;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
-
                 o = PyObject_New(PyObject, &Foo_Type);
                 init_foo(o);
                 Py_DECREF(o);   /* calls dealloc_foo immediately */
@@ -1070,8 +1062,7 @@
             ("getCounter", "METH_VARARGS",
              """
                 return PyInt_FromLong(foo_counter);
-             """)], prologue=
-            """
+             """)], prologue="""
             typedef struct {
                 PyObject_HEAD
                 int someval[99];
@@ -1105,6 +1096,14 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            """, more_init="""
+                Foo_Type.tp_basicsize = sizeof(FooObject);
+                Foo_Type.tp_dealloc = &dealloc_foo;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES
+                                    | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = &new_foo;
+                Foo_Type.tp_free = &PyObject_Del;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             """)
         Foo = module.fetchFooType()
         assert module.getCounter() == 1010
@@ -1146,14 +1145,10 @@
            ("new_obj", "METH_NOARGS",
             '''
                 PyObject *obj;
-                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                Foo_Type.tp_call = &my_tp_call;
-                if (PyType_Ready(&Foo_Type) < 0) return NULL;
                 obj = PyObject_New(PyObject, &Foo_Type);
                 return obj;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyObject *
             my_tp_call(PyObject *self, PyObject *args, PyObject *kwds)
             {
@@ -1163,6 +1158,10 @@
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.foo",
             };
+            ''', more_init='''
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                Foo_Type.tp_call = &my_tp_call;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
             ''')
         x = module.new_obj()
         assert x() == 42
@@ -1172,18 +1171,18 @@
         module = self.import_extension('foo', [
            ("getMetaClass", "METH_NOARGS",
             '''
-                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
-                FooType_Type.tp_base = &PyType_Type;
-                if (PyType_Ready(&FooType_Type) < 0) return NULL;
                 Py_INCREF(&FooType_Type);
                 return (PyObject *)&FooType_Type;
             '''
-            )],
-            '''
+            )], prologue='''
             static PyTypeObject FooType_Type = {
                 PyVarObject_HEAD_INIT(NULL, 0)
                 "foo.Type",
             };
+            ''', more_init='''
+                FooType_Type.tp_flags = Py_TPFLAGS_DEFAULT;
+                FooType_Type.tp_base = &PyType_Type;
+                if (PyType_Ready(&FooType_Type) < 0) INITERROR;
             ''')
         FooType = module.getMetaClass()
         if not self.runappdirect:
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -240,7 +240,7 @@
 def update_all_slots(space, w_type, pto):
     # fill slots in pto
     # Not very sure about it, but according to
-    # test_call_tp_dealloc_when_created_from_python, we should not
+    # test_call_tp_dealloc, we should not
     # overwrite slots that are already set: these ones are probably
     # coming from a parent C type.
 
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -120,7 +120,7 @@
 
 class DirectStreamWriter(StreamReaderWriter):
     def write(self, data):
-        self.file.do_direct_write(data)
+        self.file.direct_write_str(data)
 
 class DirectStreamReader(StreamReaderWriter):
     def read(self, n):
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -130,9 +130,12 @@
 
 
 class methodcaller(object):
-    def __init__(self, method_name, *args, **kwargs):
+    def __init__(*args, **kwargs):
+        if len(args) < 2:
+            raise TypeError("methodcaller() called with not enough arguments")
+        self, method_name = args[:2]
         self._method_name = method_name
-        self._args = args
+        self._args = args[2:]
         self._kwargs = kwargs
 
     def __call__(self, obj):
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -244,6 +244,13 @@
         assert methodcaller("method", 4, 5)(x) == (4, 5)
         assert methodcaller("method", 4, arg2=42)(x) == (4, 42)
 
+    def test_methodcaller_self(self):
+        from operator import methodcaller
+        class X:
+            def method(myself, self):
+                return self * 6
+        assert methodcaller("method", self=7)(X()) == 42
+
     def test_index(self):
         import operator
         assert operator.index(42) == 42
diff --git a/pypy/module/select/conftest.py b/pypy/module/select/conftest.py
deleted file mode 100644
--- a/pypy/module/select/conftest.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import py
-
-def pytest_collect_directory():
-    py.test.importorskip("ctypes")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/support.py b/pypy/module/test_lib_pypy/ctypes_tests/support.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/support.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/support.py
@@ -2,8 +2,6 @@
 import sys
 import ctypes
 
-py.test.importorskip("ctypes", "1.0.2")
-
 try:
     import _rawffi
 except ImportError:
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_errno.py
@@ -3,7 +3,10 @@
 import ctypes
 from _ctypes import function
 
-_rawffi = py.test.importorskip("_rawffi")
+try:
+    import _rawffi
+except ImportError:
+    py.test.skip("app-level test only for PyPy")
 
 class TestErrno:
 
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
--- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py
+++ b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
@@ -9,7 +9,9 @@
 
 def pytest_funcarg__child(request):
     try:
-        pexpect = pytest.importorskip('pexpect')
+        import pexpect
+    except ImportError:
+        pytest.skip("no pexpect module")
     except SyntaxError:
         pytest.skip('pexpect wont work on py3k')
     child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -330,6 +330,9 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -626,6 +626,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -83,8 +83,11 @@
     def import_extension(self, modname, functions, prologue="",
             include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
         body = prologue + make_methods(functions, modname)
-        init = """Py_InitModule("%s", methods);""" % (modname,)
+        init = """Py_InitModule("%s", methods);
+               """ % (modname,)
         if more_init:
+            init += """#define INITERROR return
+                    """
             init += more_init
         return self.import_module(
             name=modname, init=init, body=body, include_dirs=include_dirs,
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -613,8 +613,11 @@
             return
         cs = effectinfo.call_shortcut
         ptr_box = op.getarg(1 + cs.argnum)
-        value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                       raw=(ptr_box.type == 'i'))
+        if cs.fielddescr is not None:
+            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
+                                           raw=(ptr_box.type == 'i'))
+        else:
+            value_box = ptr_box
         self.replace_op_with(op, ResOperation(cond_call_opnum,
                                               [value_box] + op.getarglist(),
                                               descr=descr))
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -88,6 +88,13 @@
         call_shortcut_descr = get_call_descr(self.gc_ll_descr,
             [lltype.Ptr(T)], lltype.Signed,
             effectinfo)
+        call_shortcut_2 = CallShortcut(0, None)
+        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
+                                EffectInfo.EF_RANDOM_EFFECTS,
+                                call_shortcut=call_shortcut_2)
+        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
+            [lltype.Signed], lltype.Signed,
+            effectinfo_2)
         #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
@@ -1451,3 +1458,14 @@
             i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
             jump(i1)
         """)
+
+    def test_handle_call_shortcut_2(self):
+        self.check_rewrite("""
+            [i0]
+            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """, """
+            [i0]
+            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
+            jump(i1)
+        """)
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -388,15 +388,23 @@
                    return y.field          if r: return r
         """
         block = graph.startblock
-        if len(block.operations) == 0:
-            return
-        op = block.operations[0]
-        if op.opname != 'getfield':
-            return
-        [v_inst, c_fieldname] = op.args
+        operations = block.operations
+        c_fieldname = None
+        if not operations:
+            v_inst = v_result = block.exitswitch
+        else:
+            op = operations[0]
+            if len(op.args) == 0:
+                return
+            if op.opname != 'getfield':  # check for this form:
+                v_inst = op.args[0]      #     if y is not None;
+                v_result = v_inst        #          return y
+            else:
+                operations = operations[1:]
+                [v_inst, c_fieldname] = op.args
+                v_result = op.result
         if not isinstance(v_inst, Variable):
             return
-        v_result = op.result
         if v_result.concretetype != graph.getreturnvar().concretetype:
             return
         if v_result.concretetype == lltype.Void:
@@ -409,7 +417,7 @@
         PSTRUCT = v_inst.concretetype
         v_check = v_result
         fastcase = True
-        for op in block.operations[1:]:
+        for op in operations:
             if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
                     and v_check is op.args[0]):
                 v_check = op.result
@@ -439,5 +447,8 @@
         if not link.target.is_final_block():
             return
 
-        fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        if c_fieldname is not None:
+            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
+        else:
+            fielddescr = None
         return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -424,6 +424,21 @@
             r = b.foobar = 123
         return r
 
+    def f6(b):
+        if b is not None:
+            return b
+        return B()
+
+    def f7(c, a):
+        if a:
+            return a
+        return 123
+
+    def b_or_none(c):
+        if c > 15:
+            return B()
+        return None
+
     def f(a, c):
         b = B()
         f1(a, b, c)
@@ -432,6 +447,8 @@
         r = lltype.malloc(RAW, flavor='raw')
         f4(r)
         f5(b)
+        f6(b_or_none(c))
+        f7(c, a)
 
     rtyper = support.annotate(f, [10, 20])
     f1_graph = rtyper.annotator.translator._graphof(f1)
@@ -444,6 +461,10 @@
     assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
     f5_graph = rtyper.annotator.translator._graphof(f5)
     assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
+    f6_graph = rtyper.annotator.translator._graphof(f6)
+    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
+    f7_graph = rtyper.annotator.translator._graphof(f7)
+    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
 
 def test_cant_find_call_shortcut():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1874,7 +1874,9 @@
 # Support for f... and ...at families of POSIX functions
 
 class CConfig:
-    _compilation_info_ = eci
+    _compilation_info_ = eci.merge(ExternalCompilationInfo(
+        includes=['sys/statvfs.h'],
+    ))
     for _name in """faccessat fchdir fchmod fchmodat fchown fchownat fexecve
             fdopendir fpathconf fstat fstatat fstatvfs ftruncate
             futimens futimes futimesat linkat chflags lchflags lchmod lchown
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -173,6 +173,12 @@
     state = State()
 
 HAS_CLOCK_GETTIME = (CLOCK_MONOTONIC is not None)
+if sys.platform == 'darwin':
+    HAS_CLOCK_GETTIME = False
+    # ^^^ https://bitbucket.org/pypy/pypy/issues/2432 and others
+    # (change it manually if you *know* you want to build and run on
+    # OS/X 10.12 or later)
+
 if HAS_CLOCK_GETTIME:
     # Linux and other POSIX systems with clock_gettime()
     # TIMESPEC:
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -78,7 +78,6 @@
             self.log.info(s)
         except:
             self.seed = 0
-        self.order = None
 
     def getconfig(self):
         return self.annotator.translator.config
@@ -218,15 +217,9 @@
                 r = random.Random(self.seed)
                 r.shuffle(pending)
 
-            if self.order:
-                tracking = self.order(self.annotator, pending)
-            else:
-                tracking = lambda block: None
-
             previous_percentage = 0
             # specialize all blocks in the 'pending' list
             for block in pending:
-                tracking(block)
                 blockcount += 1
                 self.specialize_block(block)
                 self.already_seen[block] = True

From pypy.commits at gmail.com  Tue Nov 22 15:44:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 12:44:47 -0800 (PST)
Subject: [pypy-commit] pypy default: Remove obsolete reimplementations of
 stdlib functions
Message-ID: <5834ae3f.05371c0a.1045a.a3ec@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88554:ccde5f15328f
Date: 2016-11-22 20:42 +0000
http://bitbucket.org/pypy/pypy/changeset/ccde5f15328f/

Log:	Remove obsolete reimplementations of stdlib functions

diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -1,6 +1,7 @@
 """Float constants"""
 
 import math, struct
+from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p, expm1
 
 from rpython.annotator.model import SomeString, SomeChar
 from rpython.rlib import objectmodel, unroll
@@ -184,104 +185,6 @@
 INFINITY = 1e200 * 1e200
 NAN = abs(INFINITY / INFINITY)    # bah, INF/INF gives us -NAN?
 
-try:
-    # Try to get math functions added in 2.6.
-    from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p
-except ImportError:
-    @not_rpython
-    def isinf(x):
-        return x == INFINITY or x == -INFINITY
-
-    @not_rpython
-    def isnan(v):
-        return v != v
-
-    @not_rpython
-    def copysign(x, y):
-        """Return x with the sign of y"""
-        if x < 0.:
-            x = -x
-        if y > 0. or (y == 0. and math.atan2(y, -1.) > 0.):
-            return x
-        else:
-            return -x
-
-    _2_to_m28 = 3.7252902984619141E-09; # 2**-28
-    _2_to_p28 = 268435456.0; # 2**28
-    _ln2 = 6.93147180559945286227E-01
-
-    @not_rpython
-    def acosh(x):
-        if isnan(x):
-            return NAN
-        if x < 1.:
-            raise ValueError("math domain error")
-        if x >= _2_to_p28:
-            if isinf(x):
-                return x
-            else:
-                return math.log(x) + _ln2
-        if x == 1.:
-            return 0.
-        if x >= 2.:
-            t = x * x
-            return math.log(2. * x - 1. / (x + math.sqrt(t - 1.0)))
-        t = x - 1.0
-        return log1p(t + math.sqrt(2. * t + t * t))
-
-    @not_rpython
-    def asinh(x):
-        absx = abs(x)
-        if not isfinite(x):
-            return x
-        if absx < _2_to_m28:
-            return x
-        if absx > _2_to_p28:
-            w = math.log(absx) + _ln2
-        elif absx > 2.:
-            w = math.log(2. * absx + 1. / (math.sqrt(x * x + 1.) + absx))
-        else:
-            t = x * x
-            w = log1p(absx + t / (1. + math.sqrt(1. + t)))
-        return copysign(w, x)
-
-    @not_rpython
-    def atanh(x):
-        if isnan(x):
-            return x
-        absx = abs(x)
-        if absx >= 1.:
-            raise ValueError("math domain error")
-        if absx < _2_to_m28:
-            return x
-        if absx < .5:
-            t = absx + absx
-            t = .5 * log1p(t + t * absx / (1. - absx))
-        else:
-            t = .5 * log1p((absx + absx) / (1. - absx))
-        return copysign(t, x)
-
-    @not_rpython
-    def log1p(x):
-        if abs(x) < DBL_EPSILON // 2.:
-            return x
-        elif -.5 <= x <= 1.:
-            y = 1. + x
-            return math.log(y) - ((y - 1.) - x) / y
-        else:
-            return math.log(1. + x)
-
-try:
-    from math import expm1 # Added in Python 2.7.
-except ImportError:
-    @not_rpython
-    def expm1(x):
-        if abs(x) < .7:
-            u = math.exp(x)
-            if u == 1.:
-                return x
-            return (u - 1.) * x / math.log(u)
-        return math.exp(x) - 1.
 
 def log2(x):
     # Uses an algorithm that should:

From pypy.commits at gmail.com  Tue Nov 22 17:10:02 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 14:10:02 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: annotator fixes
Message-ID: <5834c23a.44e61c0a.ca147.bf33@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: space-newtext
Changeset: r88555:8dfca1a083c6
Date: 2016-11-22 14:08 -0800
http://bitbucket.org/pypy/pypy/changeset/8dfca1a083c6/

Log:	annotator fixes

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -217,9 +217,6 @@
 class NumericTypeConverterMixin(object):
     _mixin_ = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(obj)
-
     def convert_argument_libffi(self, space, w_obj, address, call_local):
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -78,8 +78,8 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(obj)
+    #def _wrap_object(self, space, obj):
+    #    return getattr(space, self.wrapper)(obj)
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
@@ -102,8 +102,8 @@
         self.item = self._unwrap_object(space, w_item)
         self.do_assign = True
 
-    def _wrap_object(self, space, obj):
-        return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
+    #def _wrap_object(self, space, obj):
+    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
 
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
diff --git a/pypy/module/cppyy/ffitypes.py b/pypy/module/cppyy/ffitypes.py
--- a/pypy/module/cppyy/ffitypes.py
+++ b/pypy/module/cppyy/ffitypes.py
@@ -2,6 +2,7 @@
 
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rlib.rbigint import rbigint
 from rpython.rlib import jit_libffi, rfloat
 
 # Mixins to share between converter and executor classes (in converter.py and
@@ -9,18 +10,6 @@
 # sets of jit_libffi, rffi, and different space unwrapping calls. To get the
 # right mixin, a non-RPython function typeid() is used.
 
-class BaseIntTypeMixin(object):
-    _mixin_     = True
-    _immutable_fields_ = ['wrapper']
-
-    wrapper     = 'newint'
-
-class BaseLongTypeMixin(object):
-    _mixin_     = True
-    _immutable_fields_ = ['wrapper']
-
-    wrapper     = 'newlong'
-
 class BoolTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
@@ -29,6 +18,9 @@
     c_type      = rffi.UCHAR
     c_ptrtype   = rffi.UCHARP
 
+    def _wrap_object(self, space, obj):
+        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
+
     def _unwrap_object(self, space, w_obj):
         arg = space.c_int_w(w_obj)
         if arg != False and arg != True:
@@ -36,9 +28,6 @@
                         "boolean value should be bool, or integer 1 or 0")
         return arg
 
-    def _wrap_object(self, space, obj):
-        return space.newbool(bool(ord(rffi.cast(rffi.CHAR, obj))))
-
 class CharTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
@@ -46,7 +35,9 @@
     libffitype  = jit_libffi.types.schar
     c_type      = rffi.CHAR
     c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.CHARP
-    wrapper     = 'newbytes'
+
+    def _wrap_object(self, space, obj):
+        return space.newbytes(obj)
 
     def _unwrap_object(self, space, w_value):
         # allow int to pass to char and make sure that str is of length 1
@@ -64,6 +55,15 @@
                         "char expected, got string of size %d", len(value))
         return value[0] # turn it into a "char" to the annotator
 
+class BaseIntTypeMixin(object):
+    _mixin_     = True
+
+    def _wrap_object(self, space, obj):
+        return space.newint(rffi.cast(rffi.INT, obj))
+
+    def _unwrap_object(self, space, w_obj):
+        return rffi.cast(self.c_type, space.c_int_w(w_obj))
+
 class ShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
@@ -72,9 +72,6 @@
     c_type      = rffi.SHORT
     c_ptrtype   = rffi.SHORTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(rffi.SHORT, space.int_w(w_obj))
-
 class UShortTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
@@ -83,9 +80,6 @@
     c_type      = rffi.USHORT
     c_ptrtype   = rffi.USHORTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(self.c_type, space.int_w(w_obj))
-
 class IntTypeMixin(BaseIntTypeMixin):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
@@ -94,10 +88,7 @@
     c_type      = rffi.INT
     c_ptrtype   = rffi.INTP
 
-    def _unwrap_object(self, space, w_obj):
-        return rffi.cast(self.c_type, space.c_int_w(w_obj))
-
-class UIntTypeMixin(BaseLongTypeMixin):
+class UIntTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -105,10 +96,13 @@
     c_type      = rffi.UINT
     c_ptrtype   = rffi.UINTP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(self.c_type, space.uint_w(w_obj))
 
-class LongTypeMixin(BaseLongTypeMixin):
+class LongTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -116,12 +110,13 @@
     c_type      = rffi.LONG
     c_ptrtype   = rffi.LONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
-# TODO: check ULong limits; actually, they fail if this is
-#  an BaseLongTypeMixin (i.e. use of space.ewlong) ... why??
-class ULongTypeMixin(BaseIntTypeMixin):
+class ULongTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -129,10 +124,13 @@
     c_type      = rffi.ULONG
     c_ptrtype   = rffi.ULONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
-class LongLongTypeMixin(BaseLongTypeMixin):
+class LongLongTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -140,10 +138,13 @@
     c_type      = rffi.LONGLONG
     c_ptrtype   = rffi.LONGLONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.r_longlong_w(w_obj)
 
-class ULongLongTypeMixin(BaseLongTypeMixin):
+class ULongLongTypeMixin(object):
     _mixin_     = True
     _immutable_fields_ = ['libffitype', 'c_type', 'c_ptrtype']
 
@@ -151,6 +152,9 @@
     c_type      = rffi.ULONGLONG
     c_ptrtype   = rffi.ULONGLONGP
 
+    def _wrap_object(self, space, obj):
+        return space.newlong_from_rarith_int(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.r_ulonglong_w(w_obj)
 
@@ -176,9 +180,11 @@
     libffitype  = jit_libffi.types.double
     c_type      = rffi.DOUBLE
     c_ptrtype   = rffi.DOUBLEP
-    wrapper     = 'newfloat'
     typecode    = 'd'
 
+    def _wrap_object(self, space, obj):
+        return space.newfloat(obj)
+
     def _unwrap_object(self, space, w_obj):
         return space.float_w(w_obj)
 
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -1,7 +1,7 @@
 import py, os, sys
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.objectmodel import specialize, instantiate
-from rpython.rlib import rarithmetic, jit
+from rpython.rlib import rarithmetic, rbigint, jit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper import llinterp
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
@@ -158,11 +158,20 @@
 
     @specialize.argtype(1)
     def newint(self, obj):
-        return FakeInt(rarithmetic.intmask(obj))
+        if not isinstance(obj, int):
+            return FakeLong(rbigint.rbigint.fromrarith_int(obj))
+        return FakeInt(obj)
 
     @specialize.argtype(1)
     def newlong(self, obj):
-        return FakeLong(rarithmetic.intmask(obj))
+        return FakeLong(rbigint.rbigint.fromint(obj))
+
+    @specialize.argtype(1)
+    def newlong_from_rarith_int(self, obj):
+        return FakeLong(rbigint.rbigint.fromrarith_int(obj))
+
+    def newlong_from_rbigint(self, val):
+        return FakeLong(obj)
 
     @specialize.argtype(1)
     def newfloat(self, obj):
@@ -202,10 +211,8 @@
         return w_obj.val
 
     def uint_w(self, w_obj):
-        if isinstance(w_obj, FakeInt):
-            return rarithmetic.r_uint(w_obj.val)
         assert isinstance(w_obj, FakeLong)
-        return rarithmetic.r_uint(w_obj.val)
+        return rarithmetic.r_uint(w_obj.val.touint())
 
     def str_w(self, w_obj):
         assert isinstance(w_obj, FakeString)

From pypy.commits at gmail.com  Tue Nov 22 20:00:59 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 17:00:59 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: remove reflex/cint references
 (broken and no longer supported)
Message-ID: <5834ea4b.8d071c0a.bcd6e.e9aa@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88558:538657f12c66
Date: 2016-11-22 16:14 -0800
http://bitbucket.org/pypy/pypy/changeset/538657f12c66/

Log:	remove reflex/cint references (broken and no longer supported)

diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -3,8 +3,6 @@
 from rpython.rlib import jit
 
 import cling_capi as backend
-#import reflex_capi as backend
-#import cint_capi as backend
 
 from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\
    C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_FUNC_PTR
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
deleted file mode 100644
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ /dev/null
@@ -1,437 +0,0 @@
-import py, os, sys
-
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import W_Root
-
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib import libffi, rdynload
-from rpython.tool.udir import udir
-
-from pypy.module.cppyy.capi.capi_types import C_OBJECT
-
-
-__all__ = ['identify', 'std_string_name', 'eci', 'c_load_dictionary']
-
-pkgpath = py.path.local(__file__).dirpath().join(os.pardir)
-srcpath = pkgpath.join("src")
-incpath = pkgpath.join("include")
-
-if os.environ.get("ROOTSYS"):
-    import commands
-    (stat, incdir) = commands.getstatusoutput("root-config --incdir")
-    if stat != 0:
-        rootincpath = [os.path.join(os.environ["ROOTSYS"], "include"), py.path.local(udir)]
-        rootlibpath = [os.path.join(os.environ["ROOTSYS"], "lib64"), os.path.join(os.environ["ROOTSYS"], "lib")]
-    else:
-        rootincpath = [incdir, py.path.local(udir)]
-        rootlibpath = commands.getoutput("root-config --libdir").split()
-else:
-    rootincpath = [py.path.local(udir)]
-    rootlibpath = []
-
-def identify():
-    return 'CINT'
-
-ts_reflect = True
-ts_call    = True
-ts_memory  = False
-ts_helper  = False
-
-std_string_name = 'string'
-
-# force loading in global mode of core libraries, rather than linking with
-# them as PyPy uses various version of dlopen in various places; note that
-# this isn't going to fly on Windows (note that locking them in objects and
-# calling dlclose in __del__ seems to come too late, so this'll do for now)
-with rffi.scoped_str2charp('libCint.so') as ll_libname:
-    _cintdll = rdynload.dlopen(ll_libname, rdynload.RTLD_GLOBAL | rdynload.RTLD_NOW)
-with rffi.scoped_str2charp('libCore.so') as ll_libname:
-    _coredll = rdynload.dlopen(ll_libname, rdynload.RTLD_GLOBAL | rdynload.RTLD_NOW)
-with rffi.scoped_str2charp('libHist.so') as ll_libname:
-    _coredll = rdynload.dlopen(ll_libname, rdynload.RTLD_GLOBAL | rdynload.RTLD_NOW)
-
-eci = ExternalCompilationInfo(
-    separate_module_files=[srcpath.join("cintcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
-    includes=["cintcwrapper.h"],
-    library_dirs=rootlibpath,
-    libraries=["Hist", "Core", "Cint"],
-    use_cpp_linker=True,
-)
-
-_c_load_dictionary = rffi.llexternal(
-    "cppyy_load_dictionary",
-    [rffi.CCHARP], rdynload.DLLHANDLE,
-    releasegil=False,
-    compilation_info=eci)
-
-def c_load_dictionary(name):
-    result = _c_load_dictionary(name)
-    # ignore result: libffi.CDLL(name) either returns a handle to the already
-    # open file, or will fail as well and produce a correctly formatted error
-    return libffi.CDLL(name)
-
-
-# CINT-specific pythonizations ===============================================
-_c_charp2TString = rffi.llexternal(
-    "cppyy_charp2TString",
-    [rffi.CCHARP], C_OBJECT,
-    releasegil=ts_helper,
-    compilation_info=eci)
-def c_charp2TString(space, svalue):
-    with rffi.scoped_view_charp(svalue) as charp:
-        result = _c_charp2TString(charp)
-    return result
-_c_TString2TString = rffi.llexternal(
-    "cppyy_TString2TString",
-    [C_OBJECT], C_OBJECT,
-    releasegil=ts_helper,
-    compilation_info=eci)
-def c_TString2TString(space, cppobject):
-    return _c_TString2TString(cppobject)
-
-def _get_string_data(space, w_obj, m1, m2 = None):
-    from pypy.module.cppyy import interp_cppyy
-    obj = space.interp_w(interp_cppyy.W_CPPInstance, w_obj)
-    w_1 = obj.space.call_method(w_obj, m1)
-    if m2 is None:
-        return w_1
-    return obj.space.call_method(w_1, m2)
-
-### TF1 ----------------------------------------------------------------------
-class State(object):
-    def __init__(self, space):
-        self.tfn_pyfuncs = []
-        self.tfn_callbacks = []
-
-_create_tf1 = rffi.llexternal(
-    "cppyy_create_tf1",
-    [rffi.CCHARP, rffi.ULONG, rffi.DOUBLE, rffi.DOUBLE, rffi.INT], C_OBJECT,
-    releasegil=False,
-    compilation_info=eci)
-
- at unwrap_spec(args_w='args_w')
-def tf1_tf1(space, w_self, args_w):
-    """Pythonized version of TF1 constructor:
-    takes functions and callable objects, and allows a callback into them."""
-
-    from pypy.module.cppyy import interp_cppyy
-    tf1_class = interp_cppyy.scope_byname(space, "TF1")
-
-    # expected signature:
-    #  1. (char* name, pyfunc, double xmin, double xmax, int npar = 0)
-    argc = len(args_w)
-
-    try:
-        if argc < 4 or 5 < argc:
-            raise TypeError("wrong number of arguments")
-
-        # first argument must be a name
-        funcname = space.str_w(args_w[0])
-
-        # last (optional) argument is number of parameters
-        npar = 0
-        if argc == 5: npar = space.int_w(args_w[4])
-
-        # second argument must be a callable python object
-        w_callable = args_w[1]
-        if not space.is_true(space.callable(w_callable)):
-            raise TypeError("2nd argument is not a valid python callable")
-
-        # generate a pointer to function
-        from pypy.module._cffi_backend import newtype, ctypefunc, func
-
-        c_double  = newtype.new_primitive_type(space, 'double')
-        c_doublep = newtype.new_pointer_type(space, c_double)
-
-        # wrap the callable as the signature needs modifying
-        w_ifunc = interp_cppyy.get_interface_func(space, w_callable, npar)
-
-        w_cfunc = ctypefunc.W_CTypeFunc(space, [c_doublep, c_doublep], c_double, False)
-        w_callback = func.callback(space, w_cfunc, w_ifunc, None)
-        funcaddr = rffi.cast(rffi.ULONG, w_callback.get_closure())
-
-        # so far, so good; leaves on issue: CINT is expecting a wrapper, but
-        # we need the overload that takes a function pointer, which is not in
-        # the dictionary, hence this helper:
-        newinst = _create_tf1(space.str_w(args_w[0]), funcaddr,
-                      space.float_w(args_w[2]), space.float_w(args_w[3]), npar)
-
-        # w_self is a null-ptr bound as TF1 
-        from pypy.module.cppyy.interp_cppyy import W_CPPInstance, memory_regulator
-        cppself = space.interp_w(W_CPPInstance, w_self, can_be_None=False)
-        cppself._rawobject = newinst
-        memory_regulator.register(cppself)
-
-        # tie all the life times to the TF1 instance
-        space.setattr(w_self, space.wrap('_callback'), w_callback)
-
-        # by definition for __init__
-        return None
-
-    except (OperationError, TypeError, IndexError) as e:
-        newargs_w = args_w[1:]     # drop class
-
-    # return control back to the original, unpythonized overload
-    ol = tf1_class.get_overload("TF1")
-    return ol.call(None, newargs_w)
-
-### TTree --------------------------------------------------------------------
-_ttree_Branch = rffi.llexternal(
-    "cppyy_ttree_Branch",
-    [rffi.VOIDP, rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.INT, rffi.INT], C_OBJECT,
-    releasegil=False,
-    compilation_info=eci)
-
- at unwrap_spec(args_w='args_w')
-def ttree_Branch(space, w_self, args_w):
-    """Pythonized version of TTree::Branch(): takes proxy objects and by-passes
-    the CINT-manual layer."""
-
-    from pypy.module.cppyy import interp_cppyy
-    tree_class = interp_cppyy.scope_byname(space, "TTree")
-
-    # sigs to modify (and by-pass CINT):
-    #  1. (const char*, const char*, T**,               Int_t=32000, Int_t=99)
-    #  2. (const char*, T**,                            Int_t=32000, Int_t=99)
-    argc = len(args_w)
-
-    # basic error handling of wrong arguments is best left to the original call,
-    # so that error messages etc. remain consistent in appearance: the following
-    # block may raise TypeError or IndexError to break out anytime
-
-    try:
-        if argc < 2 or 5 < argc:
-            raise TypeError("wrong number of arguments")
-
-        tree = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=True)
-        if (tree is None) or (tree.cppclass != tree_class):
-            raise TypeError("not a TTree")
-
-        # first argument must always always be cont char*
-        branchname = space.str_w(args_w[0])
-
-        # if args_w[1] is a classname, then case 1, else case 2
-        try:
-            classname = space.str_w(args_w[1])
-            addr_idx  = 2
-            w_address = args_w[addr_idx]
-        except (OperationError, TypeError):
-            addr_idx  = 1
-            w_address = args_w[addr_idx]
-
-        bufsize, splitlevel = 32000, 99
-        if addr_idx+1 < argc: bufsize = space.c_int_w(args_w[addr_idx+1])
-        if addr_idx+2 < argc: splitlevel = space.c_int_w(args_w[addr_idx+2])
-
-        # now retrieve the W_CPPInstance and build other stub arguments
-        space = tree.space    # holds the class cache in State
-        cppinstance = space.interp_w(interp_cppyy.W_CPPInstance, w_address)
-        address = rffi.cast(rffi.VOIDP, cppinstance.get_rawobject())
-        klassname = cppinstance.cppclass.full_name()
-        vtree = rffi.cast(rffi.VOIDP, tree.get_rawobject())
-
-        # call the helper stub to by-pass CINT
-        vbranch = _ttree_Branch(vtree, branchname, klassname, address, bufsize, splitlevel)
-        branch_class = interp_cppyy.scope_byname(space, "TBranch")
-        w_branch = interp_cppyy.wrap_cppobject(space, vbranch, branch_class)
-        return w_branch
-    except (OperationError, TypeError, IndexError):
-        pass
-
-    # return control back to the original, unpythonized overload
-    ol = tree_class.get_overload("Branch")
-    return ol.call(w_self, args_w)
-
-def activate_branch(space, w_branch):
-    w_branches = space.call_method(w_branch, "GetListOfBranches")
-    for i in range(space.r_longlong_w(space.call_method(w_branches, "GetEntriesFast"))):
-        w_b = space.call_method(w_branches, "At", space.wrap(i))
-        activate_branch(space, w_b)
-    space.call_method(w_branch, "SetStatus", space.wrap(1))
-    space.call_method(w_branch, "ResetReadEntry")
-
-c_ttree_GetEntry = rffi.llexternal(
-    "cppyy_ttree_GetEntry",
-    [rffi.VOIDP, rffi.LONGLONG], rffi.LONGLONG,
-    releasegil=False,
-    compilation_info=eci)
-
- at unwrap_spec(args_w='args_w')
-def ttree_getattr(space, w_self, args_w):
-    """Specialized __getattr__ for TTree's that allows switching on/off the
-    reading of individual branchs."""
-
-    from pypy.module.cppyy import interp_cppyy
-    tree = space.interp_w(interp_cppyy.W_CPPInstance, w_self)
-
-    space = tree.space            # holds the class cache in State
-
-    # prevent recursion
-    attr = space.str_w(args_w[0])
-    if attr and attr[0] == '_':
-        raise OperationError(space.w_AttributeError, args_w[0])
-
-    # try the saved cdata (for builtin types)
-    try:
-        w_cdata = space.getattr(w_self, space.wrap('_'+attr))
-        from pypy.module._cffi_backend import cdataobj
-        cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)
-        return cdata.convert_to_object()
-    except OperationError:
-        pass
-
-    # setup branch as a data member and enable it for reading
-    w_branch = space.call_method(w_self, "GetBranch", args_w[0])
-    if not space.is_true(w_branch):
-        raise OperationError(space.w_AttributeError, args_w[0])
-    activate_branch(space, w_branch)
-
-    # figure out from where we're reading
-    entry = space.r_longlong_w(space.call_method(w_self, "GetReadEntry"))
-    if entry == -1:
-        entry = 0
-
-    # setup cache structure
-    w_klassname = space.call_method(w_branch, "GetClassName")
-    if space.is_true(w_klassname):
-        # some instance
-        klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname))
-        w_obj = klass.construct()
-        # 0x10000 = kDeleteObject; reset because we own the object
-        space.call_method(w_branch, "ResetBit", space.wrap(0x10000))
-        space.call_method(w_branch, "SetObject", w_obj)
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
-        space.setattr(w_self, args_w[0], w_obj)
-        return w_obj
-    else:
-        # builtin data
-        w_leaf = space.call_method(w_self, "GetLeaf", args_w[0])
-        space.call_method(w_branch, "GetEntry", space.wrap(entry))
-
-        # location
-        w_address = space.call_method(w_leaf, "GetValuePointer")
-        buf = space.getarg_w('s*', w_address)
-        from pypy.module._rawffi import buffer
-        assert isinstance(buf, buffer.RawFFIBuffer)
-        address = rffi.cast(rffi.CCHARP, buf.datainstance.ll_buffer)
-
-        # placeholder
-        w_typename = space.call_method(w_leaf, "GetTypeName" )
-        from pypy.module.cppyy import capi
-        typename = capi.c_resolve_name(space, space.str_w(w_typename))
-        if typename == 'bool': typename = '_Bool'
-        w_address = space.call_method(w_leaf, "GetValuePointer")
-        from pypy.module._cffi_backend import cdataobj, newtype
-        cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename))
-
-        # cache result
-        space.setattr(w_self, space.wrap('_'+attr), space.wrap(cdata))
-        return space.getattr(w_self, args_w[0])
-
-class W_TTreeIter(W_Root):
-    def __init__(self, space, w_tree):
-        from pypy.module.cppyy import interp_cppyy
-        tree = space.interp_w(interp_cppyy.W_CPPInstance, w_tree)
-        self.vtree = rffi.cast(rffi.VOIDP, tree.get_cppthis(tree.cppclass))
-        self.w_tree = w_tree
-
-        self.current  = 0
-        self.maxentry = space.r_longlong_w(space.call_method(w_tree, "GetEntriesFast"))
-
-        space = self.space = tree.space          # holds the class cache in State
-        space.call_method(w_tree, "SetBranchStatus", space.wrap("*"), space.wrap(0))
-
-    def iter_w(self):
-        return self.space.wrap(self)
-
-    def next_w(self):
-        if self.current == self.maxentry:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        # TODO: check bytes read?
-        c_ttree_GetEntry(self.vtree, self.current)
-        self.current += 1 
-        return self.w_tree
-
-W_TTreeIter.typedef = TypeDef(
-    'TTreeIter',
-    __iter__ = interp2app(W_TTreeIter.iter_w),
-    next = interp2app(W_TTreeIter.next_w),
-)
-
-def ttree_iter(space, w_self):
-    """Allow iteration over TTree's. Also initializes branch data members and
-    sets addresses, if needed."""
-    w_treeiter = W_TTreeIter(space, w_self)
-    return w_treeiter
-
-# setup pythonizations for later use at run-time
-_pythonizations = {}
-def register_pythonizations(space):
-    "NOT_RPYTHON"
-
-    allfuncs = [
-
-        ### TF1
-        tf1_tf1,
-
-        ### TTree
-        ttree_Branch, ttree_iter, ttree_getattr,
-    ]
-
-    for f in allfuncs:
-        _pythonizations[f.__name__] = space.wrap(interp2app(f))
-
-def _method_alias(space, w_pycppclass, m1, m2):
-    space.setattr(w_pycppclass, space.wrap(m1),
-                  space.getattr(w_pycppclass, space.wrap(m2)))
-
-# callback coming in when app-level bound classes have been created
-def pythonize(space, name, w_pycppclass):
-
-    if name == "TCollection":
-        _method_alias(space, w_pycppclass, "append", "Add")
-        _method_alias(space, w_pycppclass, "__len__", "GetSize")
-
-    elif name == "TF1":
-        space.setattr(w_pycppclass, space.wrap("__init__"), _pythonizations["tf1_tf1"])
-
-    elif name == "TFile":
-        _method_alias(space, w_pycppclass, "__getattr__", "Get")
-
-    elif name == "TObjString":
-        _method_alias(space, w_pycppclass, "__str__", "GetName")
-        _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "GetString")
-
-    elif name == "TString":
-        _method_alias(space, w_pycppclass, "__str__", "Data")
-        _method_alias(space, w_pycppclass, "__len__", "Length")
-        _method_alias(space, w_pycppclass, "__cmp__", "CompareTo")
-        _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "Data")
-
-    elif name == "TTree":
-        _method_alias(space, w_pycppclass, "_unpythonized_Branch", "Branch")
-
-        space.setattr(w_pycppclass, space.wrap("Branch"),      _pythonizations["ttree_Branch"])
-        space.setattr(w_pycppclass, space.wrap("__iter__"),    _pythonizations["ttree_iter"])
-        space.setattr(w_pycppclass, space.wrap("__getattr__"), _pythonizations["ttree_getattr"])
-
-    elif name[0:8] == "TVectorT":    # TVectorT<> template
-        _method_alias(space, w_pycppclass, "__len__", "GetNoElements")
-
-# destruction callback (needs better solution, but this is for CINT
-# only and should not appear outside of ROOT-specific uses)
-from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL
-
- at cpython_api([rffi.VOIDP], lltype.Void, error=CANNOT_FAIL)
-def _Py_cppyy_recursive_remove(space, cppobject):
-    from pypy.module.cppyy.interp_cppyy import memory_regulator
-    from pypy.module.cppyy.capi import C_OBJECT, C_NULL_OBJECT
-
-    obj = memory_regulator.retrieve(rffi.cast(C_OBJECT, cppobject))
-    if obj is not None:
-        memory_regulator.unregister(obj)
-        obj._rawobject = C_NULL_OBJECT
diff --git a/pypy/module/cppyy/capi/reflex_capi.py b/pypy/module/cppyy/capi/reflex_capi.py
deleted file mode 100644
--- a/pypy/module/cppyy/capi/reflex_capi.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import py, os
-
-from rpython.rlib import libffi
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-
-__all__ = ['identify', 'std_string_name', 'eci', 'c_load_dictionary']
-
-pkgpath = py.path.local(__file__).dirpath().join(os.pardir)
-srcpath = pkgpath.join("src")
-incpath = pkgpath.join("include")
-
-import commands
-(config_stat, incdir) = commands.getstatusoutput("root-config --incdir")
-
-if os.environ.get("ROOTSYS"):
-    if config_stat != 0:     # presumably Reflex-only
-        rootincpath = [os.path.join(os.environ["ROOTSYS"], "include")]
-        rootlibpath = [os.path.join(os.environ["ROOTSYS"], "lib64"), os.path.join(os.environ["ROOTSYS"], "lib")]
-    else:
-        rootincpath = [incdir]
-        rootlibpath = commands.getoutput("root-config --libdir").split()
-else:
-    if config_stat == 0:
-        rootincpath = [incdir]
-        rootlibpath = commands.getoutput("root-config --libdir").split()
-    else:
-        rootincpath = []
-        rootlibpath = []
-
-def identify():
-    return 'Reflex'
-
-ts_reflect = False
-ts_call    = 'auto'
-ts_memory  = 'auto'
-ts_helper  = 'auto'
-
-std_string_name = 'std::basic_string<char>'
-
-eci = ExternalCompilationInfo(
-    separate_module_files=[srcpath.join("reflexcwrapper.cxx")],
-    include_dirs=[incpath] + rootincpath,
-    includes=["reflexcwrapper.h"],
-    library_dirs=rootlibpath,
-    libraries=["Reflex"],
-    use_cpp_linker=True,
-)
-
-def c_load_dictionary(name):
-    return libffi.CDLL(name)
-
-
-# Reflex-specific pythonizations
-def register_pythonizations(space):
-    "NOT_RPYTHON"
-    pass
-
-def pythonize(space, name, w_pycppclass):
-    pass

From pypy.commits at gmail.com  Tue Nov 22 20:01:03 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 17:01:03 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: remove reflex/cint C++ codes
Message-ID: <5834ea4f.4dd41c0a.bd650.e8db@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88560:e814e2578cfa
Date: 2016-11-22 16:30 -0800
http://bitbucket.org/pypy/pypy/changeset/e814e2578cfa/

Log:	remove reflex/cint C++ codes

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -9,8 +9,8 @@
 # the selection of the desired backend (default is Reflex).
 
 # choose C-API access method:
-from pypy.module.cppyy.capi.loadable_capi import *
-#from pypy.module.cppyy.capi.builtin_capi import *
+#from pypy.module.cppyy.capi.loadable_capi import *
+from pypy.module.cppyy.capi.builtin_capi import *
 
 from pypy.module.cppyy.capi.capi_types import C_OBJECT,\
     C_NULL_TYPE, C_NULL_OBJECT
diff --git a/pypy/module/cppyy/include/cintcwrapper.h b/pypy/module/cppyy/include/cintcwrapper.h
deleted file mode 100644
--- a/pypy/module/cppyy/include/cintcwrapper.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef CPPYY_CINTCWRAPPER
-#define CPPYY_CINTCWRAPPER
-
-#include "capi.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif // ifdef __cplusplus
-
-    /* misc helpers */
-    void* cppyy_load_dictionary(const char* lib_name);
-
-    /* pythonization helpers */
-    cppyy_object_t cppyy_create_tf1(const char* funcname, unsigned long address,
-        double xmin, double xmax, int npar);
-
-    cppyy_object_t cppyy_ttree_Branch(
-        void* vtree, const char* branchname, const char* classname,
-        void* addobj, int bufsize, int splitlevel);
-
-    long long cppyy_ttree_GetEntry(void* vtree, long long entry);
-
-    cppyy_object_t cppyy_charp2TString(const char* str);
-    cppyy_object_t cppyy_TString2TString(cppyy_object_t ptr);
-
-#ifdef __cplusplus
-}
-#endif // ifdef __cplusplus
-
-#endif // ifndef CPPYY_CINTCWRAPPER
diff --git a/pypy/module/cppyy/include/reflexcwrapper.h b/pypy/module/cppyy/include/reflexcwrapper.h
deleted file mode 100644
--- a/pypy/module/cppyy/include/reflexcwrapper.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef CPPYY_REFLEXCWRAPPER
-#define CPPYY_REFLEXCWRAPPER
-
-#include "capi.h"
-
-#endif // ifndef CPPYY_REFLEXCWRAPPER
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
deleted file mode 100644
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ /dev/null
@@ -1,1074 +0,0 @@
-#include "cppyy.h"
-#include "cintcwrapper.h"
-
-#include "TROOT.h"
-#include "TError.h"
-#include "TList.h"
-#include "TSystem.h"
-
-#include "TApplication.h"
-#include "TInterpreter.h"
-#include "TVirtualMutex.h"
-#include "Getline.h"
-
-#include "TBaseClass.h"
-#include "TClass.h"
-#include "TClassEdit.h"
-#include "TClassRef.h"
-#include "TClassTable.h"
-#include "TDataMember.h"
-#include "TFunction.h"
-#include "TGlobal.h"
-#include "TMethod.h"
-#include "TMethodArg.h"
-
-// for pythonization
-#include "TTree.h"
-#include "TBranch.h"
-#include "TF1.h"
-#include "TString.h"
-
-#include "Api.h"
-
-#include <assert.h>
-#include <string.h>
-#include <map>
-#include <sstream>
-#include <string>
-#include <utility>
-
-// for recursive_remove callback
-#include "pypy_macros.h"
-
-
-/* ROOT/CINT internals --------------------------------------------------- */
-extern long G__store_struct_offset;
-extern "C" void G__LockCriticalSection();
-extern "C" void G__UnlockCriticalSection();
-
-namespace {
-
-class Cppyy_OpenedTClass : public TDictionary {
-public:
-    mutable TObjArray* fStreamerInfo;    //Array of TVirtualStreamerInfo
-    mutable std::map<std::string, TObjArray*>* fConversionStreamerInfo; //Array of the streamer infos derived from another class.
-    TList*             fRealData;       //linked list for persistent members including base classes
-    TList*             fBase;           //linked list for base classes
-    TList*             fData;           //linked list for data members
-    TList*             fMethod;         //linked list for methods
-    TList*             fAllPubData;     //all public data members (including from base classes)
-    TList*             fAllPubMethod;   //all public methods (including from base classes)
-};
-
-// memory regulation (cppyy_recursive_remove is generated as a cpyext capi call)
-extern "C" void _Py_cppyy_recursive_remove(void*);
-
-class Cppyy_MemoryRegulator : public TObject {
-public:
-    virtual void RecursiveRemove(TObject* object) {
-        _Py_cppyy_recursive_remove((void*)object);
-    }
-};
-
-} // unnamed namespace
-
-
-/* data for life time management ------------------------------------------ */
-#define GLOBAL_HANDLE 1l
-
-typedef std::vector<TClassRef> ClassRefs_t;
-static ClassRefs_t g_classrefs(1);
-
-typedef std::map<std::string, ClassRefs_t::size_type> ClassRefIndices_t;
-static ClassRefIndices_t g_classref_indices;
-
-typedef std::vector<TFunction> GlobalFuncs_t;
-static GlobalFuncs_t g_globalfuncs;
-
-typedef std::vector<TGlobal> GlobalVars_t;
-static GlobalVars_t g_globalvars;
-
-typedef std::vector<G__MethodInfo> InterpretedFuncs_t;
-static InterpretedFuncs_t g_interpreted;
-
-
-/* initialization of the ROOT system (debatable ... ) --------------------- */
-namespace {
-
-static Cppyy_MemoryRegulator s_memreg;
-
-class TCppyyApplication : public TApplication {
-public:
-    TCppyyApplication(const char* acn, Int_t* argc, char** argv, Bool_t do_load = kTRUE)
-           : TApplication(acn, argc, argv) {
-
-        // Explicitly load libMathCore as CINT will not auto load it when using
-        // one of its globals. Once moved to Cling, which should work correctly,
-        // we can remove this statement.
-        gSystem->Load("libMathCore");
-
-        if (do_load) {
-            // follow TRint to minimize differences with CINT
-            ProcessLine("#include <iostream>", kTRUE);
-            ProcessLine("#include <_string>",  kTRUE); // for std::string iostream.
-            ProcessLine("#include <DllImport.h>", kTRUE);// Defined R__EXTERN
-            ProcessLine("#include <vector>",   kTRUE); // needed because they're used within the
-            ProcessLine("#include <pair>",     kTRUE); //  core ROOT dicts and CINT won't be able
-                                                       //  to properly unload these files
-        }
-
-        // save current interpreter context
-        gInterpreter->SaveContext();
-        gInterpreter->SaveGlobalsContext();
-
-        // prevent crashes on accessing history
-        Gl_histinit((char*)"-");
-
-        // prevent ROOT from exiting python
-        SetReturnFromRun(kTRUE);
-
-        // enable auto-loader
-        gInterpreter->EnableAutoLoading();
-
-        // enable memory regulation
-        gROOT->GetListOfCleanups()->Add(&s_memreg);
-    }
-};
-
-static const char* appname = "PyPyROOT";
-
-class ApplicationStarter {
-public:
-    ApplicationStarter() {
-        // setup dummy holders for global and std namespaces
-        assert(g_classrefs.size() == (ClassRefs_t::size_type)GLOBAL_HANDLE);
-        g_classref_indices[""] = (ClassRefs_t::size_type)GLOBAL_HANDLE;
-        g_classrefs.push_back(TClassRef(""));
-        // CINT ignores std/::std, so point them to the global namespace
-        g_classref_indices["std"]   = (ClassRefs_t::size_type)GLOBAL_HANDLE;
-        g_classref_indices["::std"] = (ClassRefs_t::size_type)GLOBAL_HANDLE;
-
-        // an offset for the interpreted methods
-        g_interpreted.push_back(G__MethodInfo());
-
-        // actual application init, if necessary
-        if (!gApplication) {
-            int argc = 1;
-            char* argv[1]; argv[0] = (char*)appname;
-            gApplication = new TCppyyApplication(appname, &argc, argv, kTRUE);
-            if (!gProgName)                  // should have been set by TApplication
-                gSystem->SetProgname(appname);
-        }
-
-        // program name should've been set by TApplication; just in case ...
-        if (!gProgName) {
-            gSystem->SetProgname(appname);
-        }
-    }
-} _applicationStarter;
-
-} // unnamed namespace
-
-
-/* local helpers ---------------------------------------------------------- */
-static inline const std::string resolve_typedef(const std::string& tname) {
-    G__TypeInfo ti(tname.c_str());
-    if (!ti.IsValid())
-        return tname;
-    return TClassEdit::ShortType(TClassEdit::CleanType(ti.TrueName(), 1).c_str(), 3);
-}
-
-static inline char* cppstring_to_cstring(const std::string& name) {
-    char* name_char = (char*)malloc(name.size() + 1);
-    strcpy(name_char, name.c_str());
-    return name_char;
-}
-
-static inline char* type_cppstring_to_cstring(const std::string& tname) {
-    G__TypeInfo ti(tname.c_str());
-    std::string true_name = ti.IsValid() ? ti.TrueName() : tname;
-    return cppstring_to_cstring(true_name);
-}
-
-static inline TClassRef& type_from_handle(cppyy_type_t handle) {
-    assert((ClassRefs_t::size_type)handle < g_classrefs.size());
-    return g_classrefs[(ClassRefs_t::size_type)handle];
-}
-
-static inline TFunction* type_get_method(cppyy_type_t handle, cppyy_index_t idx) {
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass())
-        return (TFunction*)cr->GetListOfMethods()->At(idx);
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    return (TFunction*)idx;
-}
-
-static inline void fixup_args(G__param* libp) {
-    for (int i = 0; i < libp->paran; ++i) {
-        libp->para[i].ref = libp->para[i].obj.i;
-        const char partype = libp->para[i].type;
-        switch (partype) {
-        case 'p': {
-            libp->para[i].obj.i = (long)&libp->para[i].ref;
-            break;
-        }
-        case 'r': {
-            libp->para[i].ref = (long)&libp->para[i].obj.i;
-            break;
-        }
-        case 'f': {
-            float val = libp->para[i].obj.fl;
-            libp->para[i].obj.d = val;
-            break;
-        }
-        case 'F': {
-            libp->para[i].ref = (long)&libp->para[i].obj.i;
-            libp->para[i].type = 'f';
-            break;
-        }
-        case 'D': {
-            libp->para[i].ref = (long)&libp->para[i].obj.i;
-            libp->para[i].type = 'd';
-            break;
-        }
-        }
-    }
-}
-
-
-/* name to opaque C++ scope representation -------------------------------- */
-int cppyy_num_scopes(cppyy_scope_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        /* not supported as CINT does not store classes hierarchically */
-        return 0;
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    return gClassTable->Classes();
-}
-
-char* cppyy_scope_name(cppyy_scope_t handle, int iscope) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        /* not supported as CINT does not store classes hierarchically */
-        assert(!"scope name lookup not supported on inner scopes");
-        return 0;
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    std::string name = gClassTable->At(iscope);
-    if (name.find("::") == std::string::npos)
-        return cppstring_to_cstring(name);
-    return cppstring_to_cstring("");
-}
-
-char* cppyy_resolve_name(const char* cppitem_name) {
-    R__LOCKGUARD2(gCINTMutex);
-    std::string tname = cppitem_name;
-
-    // global namespace?
-    if (tname.empty())
-        return cppstring_to_cstring(cppitem_name);
-
-    // special care needed for builtin arrays
-    std::string::size_type pos = tname.rfind("[");
-    G__TypeInfo ti(tname.substr(0, pos).c_str());
-
-    // if invalid (most likely unknown), simply return old name
-    if (!ti.IsValid())
-        return cppstring_to_cstring(cppitem_name);
-
-    // special case treatment of enum types as unsigned int (CINTism)
-    if (ti.Property() & G__BIT_ISENUM)
-        return cppstring_to_cstring("unsigned int");
-
-    // actual typedef resolution; add back array declaration portion, if needed
-    std::string rt = ti.TrueName();
-
-    // builtin STL types have fake typedefs :/
-    G__TypeInfo ti_test(rt.c_str());
-    if (!ti_test.IsValid())
-        return cppstring_to_cstring(cppitem_name);
-
-    if (pos != std::string::npos)
-        rt += tname.substr(pos, std::string::npos);
-    return cppstring_to_cstring(rt);
-}
-
-cppyy_scope_t cppyy_get_scope(const char* scope_name) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    // CINT still has trouble with std:: sometimes ... 
-    if (strncmp(scope_name, "std::", 5) == 0)
-        scope_name = &scope_name[5];
-
-    ClassRefIndices_t::iterator icr = g_classref_indices.find(scope_name);
-    if (icr != g_classref_indices.end())
-        return (cppyy_type_t)icr->second;
-
-    if (strcmp(scope_name, "#define") == 0)
-        return (cppyy_type_t)NULL;
-
-    // use TClass directly, to enable auto-loading
-    TClassRef cr(TClass::GetClass(scope_name, kTRUE, kTRUE));
-    if (!cr.GetClass())
-        return (cppyy_type_t)NULL;
-
-    if (!cr->GetClassInfo())
-        return (cppyy_type_t)NULL;
-
-    if (!G__TypeInfo(scope_name).IsValid())
-        return (cppyy_type_t)NULL;
-
-    ClassRefs_t::size_type sz = g_classrefs.size();
-    g_classref_indices[scope_name] = sz;
-    g_classrefs.push_back(TClassRef(scope_name));
-    return (cppyy_scope_t)sz;
-}
-
-cppyy_type_t cppyy_get_template(const char* template_name) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    ClassRefIndices_t::iterator icr = g_classref_indices.find(template_name);
-    if (icr != g_classref_indices.end())
-        return (cppyy_type_t)icr->second;
-
-    if (!G__defined_templateclass((char*)template_name))
-        return (cppyy_type_t)NULL;
-
-    // #include for specific, pre-compiled STL classes
-    if (strcmp(template_name, "std::map") == 0)
-        gROOT->ProcessLine("#include <map>");
-
-    // the following yields a dummy TClassRef, but its name can be queried
-    ClassRefs_t::size_type sz = g_classrefs.size();
-    g_classref_indices[template_name] = sz;
-    g_classrefs.push_back(TClassRef(template_name));
-    return (cppyy_type_t)sz;
-}
-
-cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(klass);
-    TClass* clActual = cr->GetActualClass( (void*)obj );
-    if (clActual && clActual != cr.GetClass()) {
-        // TODO: lookup through name should not be needed
-        return (cppyy_type_t)cppyy_get_scope(clActual->GetName());
-    }
-    return klass;
-}
-
-
-/* memory management ------------------------------------------------------ */
-cppyy_object_t cppyy_allocate(cppyy_type_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    return (cppyy_object_t)malloc(cr->Size());
-}
-
-void cppyy_deallocate(cppyy_type_t /*handle*/, cppyy_object_t instance) {
-    free((void*)instance);
-}
-
-void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    cr->Destructor((void*)self, true);
-}
-
-
-/* method/function dispatching -------------------------------------------- */
-static inline G__value cppyy_call_T(cppyy_method_t method,
-        cppyy_object_t self, int nargs, void* args) {
-
-    R__LOCKGUARD2(gCINTMutex);
-
-    G__param* libp = (G__param*)((char*)args - offsetof(G__param, para));
-    assert(libp->paran == nargs);
-    fixup_args(libp);
-
-    if ((InterpretedFuncs_t::size_type)method < g_interpreted.size()) {
-    // the idea here is that all these low values are invalid memory addresses,
-    // allowing the reuse of method to index the stored bytecodes
-        G__CallFunc callf;
-        callf.SetFunc(g_interpreted[(size_t)method]);
-        G__param p;      // G__param has fixed size; libp is sized to nargs
-        for (int i =0; i<nargs; ++i)
-            p.para[i] = libp->para[i];
-        p.paran = nargs;
-        callf.SetArgs(p);     // will copy p yet again
-        return callf.Execute((void*)self);
-    }
-
-    G__InterfaceMethod meth = (G__InterfaceMethod)method;
-
-    G__value result;
-    G__setnull(&result);
-
-    G__LockCriticalSection();      // CINT-level lock, is recursive
-    G__settemplevel(1);
-
-    long index = (long)&method;
-
-    // TODO: access to store_struct_offset won't work on Windows
-    long store_struct_offset = G__store_struct_offset;
-    if (self)
-        G__store_struct_offset = (long)self;
-
-    meth(&result, (char*)0, libp, 0);
-    if (self)
-        G__store_struct_offset = store_struct_offset;
-
-    if (G__get_return(0) > G__RETURN_NORMAL)
-        G__security_recover(0);    // 0 ensures silence
-
-    G__settemplevel(-1);
-    G__UnlockCriticalSection();
-
-    return result;
-}
-
-void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    cppyy_call_T(method, self, nargs, args);
-}
-
-unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (unsigned char)(bool)G__int(result);
-}
-
-char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (char)G__int(result);
-}
-
-short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (short)G__int(result);
-}
-
-int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (int)G__int(result);
-}
-
-long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return G__int(result);
-}
-
-long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return G__Longlong(result);
-}
-
-float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (float)G__double(result);
-}
-
-double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return G__double(result);
-}
-
-void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    return (void*)result.ref;
-}
-
-char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    R__LOCKGUARD2(gCINTMutex);
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    G__pop_tempobject_nodel();
-    if (result.ref && *(long*)result.ref) {
-        char* charp = cppstring_to_cstring(*(std::string*)result.ref);
-        delete (std::string*)result.ref;
-        return charp;
-    }
-    return cppstring_to_cstring("");
-}
-
-cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
-    R__LOCKGUARD2(gCINTMutex);
-    cppyy_object_t self = (cppyy_object_t)NULL;
-    if ((InterpretedFuncs_t::size_type)method >= g_interpreted.size()) {
-        G__setgvp((long)G__PVOID);
-        self = (cppyy_object_t)cppyy_call_l(method, (cppyy_object_t)NULL, nargs, args);
-    } else {
-    // for macro's/interpreted classes
-        self = cppyy_allocate(handle);
-        G__setgvp((long)self);
-        cppyy_call_T(method, self, nargs, args);
-    }
-    G__setgvp((long)G__PVOID);
-    return self;
-}
-
-cppyy_object_t cppyy_call_o(cppyy_type_t method, cppyy_object_t self, int nargs, void* args,
-                  cppyy_type_t /*result_type*/ ) {
-    R__LOCKGUARD2(gCINTMutex);
-    G__value result = cppyy_call_T(method, self, nargs, args);
-    G__pop_tempobject_nodel();
-    return (cppyy_object_t)G__int(result);
-}
-
-cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /*handle*/, cppyy_index_t /*idx*/) {
-    return (cppyy_methptrgetter_t)NULL;
-}
-
-
-/* handling of function argument buffer ----------------------------------- */
-void* cppyy_allocate_function_args(int nargs) {
-    assert(sizeof(CPPYY_G__value) == sizeof(G__value));
-    G__param* libp = (G__param*)malloc(
-        offsetof(G__param, para) + nargs*sizeof(CPPYY_G__value));
-    libp->paran = (int)nargs;
-    for (int i = 0; i < nargs; ++i)
-        libp->para[i].type = 'l';
-    return (void*)libp->para;
-}
-
-void cppyy_deallocate_function_args(void* args) {
-    free((char*)args - offsetof(G__param, para));
-}
-
-size_t cppyy_function_arg_sizeof() {
-    return sizeof(CPPYY_G__value);
-}
-
-size_t cppyy_function_arg_typeoffset() {
-    return offsetof(CPPYY_G__value, type);
-}
-
-
-/* scope reflection information ------------------------------------------- */
-int cppyy_is_namespace(cppyy_scope_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetClassInfo())
-        return cr->Property() & G__BIT_ISNAMESPACE;
-    if (handle == (cppyy_scope_t)GLOBAL_HANDLE)
-        return true;
-    return false;
-}
-
-int cppyy_is_enum(const char* type_name) {
-    R__LOCKGUARD2(gCINTMutex);
-    G__TypeInfo ti(type_name);
-    return (ti.Property() & G__BIT_ISENUM);
-}
-
-
-/* type/class reflection information -------------------------------------- */
-char* cppyy_final_name(cppyy_type_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetClassInfo()) {
-        std::string true_name = G__TypeInfo(cr->GetName()).TrueName();
-        std::string::size_type pos = true_name.rfind("::");
-        if (pos != std::string::npos)
-            return cppstring_to_cstring(true_name.substr(pos+2, std::string::npos));
-        return cppstring_to_cstring(true_name);
-    }
-    return cppstring_to_cstring(cr.GetClassName());
-}
-
-char* cppyy_scoped_final_name(cppyy_type_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetClassInfo()) {
-        std::string true_name = G__TypeInfo(cr->GetName()).TrueName();
-        return cppstring_to_cstring(true_name);
-    }
-    return cppstring_to_cstring(cr.GetClassName());
-}
-
-int cppyy_has_complex_hierarchy(cppyy_type_t handle) {
-// as long as no fast path is supported for CINT, calculating offsets (which
-// are cached by the JIT) is not going to hurt 
-    return 1;
-}
-
-int cppyy_num_bases(cppyy_type_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetListOfBases() != 0)
-        return cr->GetListOfBases()->GetSize();
-    return 0;
-}
-
-char* cppyy_base_name(cppyy_type_t handle, int base_index) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    TBaseClass* b = (TBaseClass*)cr->GetListOfBases()->At(base_index);
-    return type_cppstring_to_cstring(b->GetName());
-}
-
-int cppyy_is_subtype(cppyy_type_t derived_handle, cppyy_type_t base_handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& derived_type = type_from_handle(derived_handle);
-    TClassRef& base_type = type_from_handle(base_handle);
-    return derived_type->GetBaseClass(base_type) != 0;
-}
-
-ptrdiff_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
-                       cppyy_object_t address, int /* direction */) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    // WARNING: CINT can not handle actual dynamic casts!
-    TClassRef& derived_type = type_from_handle(derived_handle);
-    TClassRef& base_type = type_from_handle(base_handle);
-
-    long offset = 0;
-
-    if (derived_type && base_type) {
-        G__ClassInfo* base_ci    = (G__ClassInfo*)base_type->GetClassInfo();
-        G__ClassInfo* derived_ci = (G__ClassInfo*)derived_type->GetClassInfo();
-
-        if (base_ci && derived_ci) {
-#ifdef WIN32
-            // Windows cannot cast-to-derived for virtual inheritance
-            // with CINT's (or Reflex's) interfaces.
-            long baseprop = derived_ci->IsBase(*base_ci);
-            if (!baseprop || (baseprop & G__BIT_ISVIRTUALBASE))
-                offset = derived_type->GetBaseClassOffset(base_type);
-            else
-#endif
-                offset = G__isanybase(base_ci->Tagnum(), derived_ci->Tagnum(), (long)address);
-         } else {
-             offset = derived_type->GetBaseClassOffset(base_type);
-         }
-    }
-
-    return (ptrdiff_t) offset;   // may be negative (will roll over)
-}
-
-
-/* method/function reflection information --------------------------------- */
-int cppyy_num_methods(cppyy_scope_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetListOfMethods())
-        return cr->GetListOfMethods()->GetSize();
-    else if (handle == (cppyy_scope_t)GLOBAL_HANDLE) {
-        if (g_globalfuncs.empty()) {
-            TCollection* funcs = gROOT->GetListOfGlobalFunctions(kTRUE);
-	    g_globalfuncs.reserve(funcs->GetSize());
-
-            TIter ifunc(funcs);
-
-            TFunction* func = 0;
-            while ((func = (TFunction*)ifunc.Next())) {
-                if (strcmp(func->GetName(), "G__ateval") != 0)
-                    g_globalfuncs.push_back(*func);
-            }
-        }
-	return (int)g_globalfuncs.size();
-    }
-    return 0;
-}
-
-cppyy_index_t cppyy_method_index_at(cppyy_scope_t handle, int imeth) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass())
-        return (cppyy_index_t)imeth;
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    return (cppyy_index_t)&g_globalfuncs[imeth];
-}
-
-cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t handle, const char* name) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    std::vector<cppyy_index_t> result;
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        gInterpreter->UpdateListOfMethods(cr.GetClass());
-        int imeth = 0;
-        TFunction* func;
-        TIter next(cr->GetListOfMethods());
-        while ((func = (TFunction*)next())) {
-            if (strcmp(name, func->GetName()) == 0) {
-                if (func->Property() & G__BIT_ISPUBLIC)
-                    result.push_back((cppyy_index_t)imeth);
-            }
-            ++imeth;
-        }
-    } else if (handle == (cppyy_scope_t)GLOBAL_HANDLE) {
-        TCollection* funcs = gROOT->GetListOfGlobalFunctions(kTRUE);
-        TFunction* func = 0;
-        TIter ifunc(funcs);
-        while ((func = (TFunction*)ifunc.Next())) {
-            if (strcmp(name, func->GetName()) == 0) {
-                g_globalfuncs.push_back(*func);
-                result.push_back((cppyy_index_t)func); 
-            }
-        }
-    }
-
-    if (result.empty())
-        return (cppyy_index_t*)0;
-
-    cppyy_index_t* llresult = (cppyy_index_t*)malloc(sizeof(cppyy_index_t)*(result.size()+1));
-    for (int i = 0; i < (int)result.size(); ++i) llresult[i] = result[i];
-    llresult[result.size()] = -1;
-    return llresult;
-}
-
-
-char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TFunction* f = type_get_method(handle, idx);
-    std::string name = f->GetName();
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cppyy_is_constructor(handle, idx))
-        return cppstring_to_cstring(name);
-    if (cppyy_method_is_template(handle, idx))
-       return cppstring_to_cstring(name.substr(0, name.find('<')));
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cppyy_is_constructor(handle, idx))
-        return cppstring_to_cstring("constructor");
-    TFunction* f = type_get_method(handle, idx);
-    return type_cppstring_to_cstring(f->GetReturnTypeName());
-}
-
-int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TFunction* f = type_get_method(handle, idx);
-    return f->GetNargs();
-}
-
-int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TFunction* f = type_get_method(handle, idx);
-    return f->GetNargs() - f->GetNargsOpt();
-}
-
-char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t idx, int arg_index) {
-    R__LOCKGUARD2(gCINTMutex);
-    TFunction* f = type_get_method(handle, idx);
-    TMethodArg* arg = (TMethodArg*)f->GetListOfMethodArgs()->At(arg_index);
-    return type_cppstring_to_cstring(arg->GetFullTypeName());
-}
-
-char* cppyy_method_arg_default(cppyy_scope_t /*handle*/, cppyy_index_t /*idx*/, int /*arg_index*/) {
-    /* unused: libffi does not work with CINT back-end */
-    return cppstring_to_cstring("");
-}
-
-char* cppyy_method_signature(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    TFunction* f = type_get_method(handle, idx);
-    std::ostringstream sig;
-    if (cr.GetClass() && cr->GetClassInfo()
-        && strcmp(f->GetName(), ((G__ClassInfo*)cr->GetClassInfo())->Name()) != 0)
-        sig << f->GetReturnTypeName() << " ";
-    sig << cr.GetClassName() << "::" << f->GetName() << "(";
-    int nArgs = f->GetNargs();
-    for (int iarg = 0; iarg < nArgs; ++iarg) {
-        sig << ((TMethodArg*)f->GetListOfMethodArgs()->At(iarg))->GetFullTypeName();
-        if (iarg != nArgs-1)
-            sig << ", ";
-    }
-    sig << ")" << std::ends;
-    return cppstring_to_cstring(sig.str());
-}
-
-
-int cppyy_method_is_template(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    TFunction* f = type_get_method(handle, idx);
-    std::string name = f->GetName();
-    return (name[name.size()-1] == '>') && (name.find('<') != std::string::npos);
-}
-
-int cppyy_method_num_template_args(cppyy_scope_t /*handle*/, cppyy_index_t /*idx*/) {
-// TODO: somehow count from the actual arguments
-    return 1;
-}
-
-char* cppyy_method_template_arg_name(
-        cppyy_scope_t handle, cppyy_index_t idx, cppyy_index_t /*iarg*/) {
-    R__LOCKGUARD2(gCINTMutex);
-// TODO: return only the name for the requested arg
-    TClassRef& cr = type_from_handle(handle);
-    TFunction* f = type_get_method(handle, idx);
-    std::string name = f->GetName();
-    std::string::size_type pos = name.find('<');
-    return cppstring_to_cstring(resolve_typedef(name.substr(pos+1, name.size()-pos-2)));
-}
-
-
-cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& cr = type_from_handle(handle);
-    TFunction* f = type_get_method(handle, idx);
-    if (cr && cr.GetClass() && !cr->IsLoaded()) {
-        G__ClassInfo* gcl = (G__ClassInfo*)cr->GetClassInfo();
-        if (gcl) {
-            long offset;
-            std::ostringstream sig;
-            int nArgs = f->GetNargs();
-            for (int iarg = 0; iarg < nArgs; ++iarg) {
-                sig << ((TMethodArg*)f->GetListOfMethodArgs()->At(iarg))->GetFullTypeName();
-                if (iarg != nArgs-1) sig << ", ";
-            }
-            G__MethodInfo gmi = gcl->GetMethod(
-                f->GetName(), sig.str().c_str(), &offset, G__ClassInfo::ExactMatch);
-            cppyy_method_t method = (cppyy_method_t)g_interpreted.size();
-            g_interpreted.push_back(gmi);
-            return method;
-        }
-    }
-    cppyy_method_t method = (cppyy_method_t)f->InterfaceMethod();
-    return method;
-}
-
-cppyy_index_t cppyy_get_global_operator(cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& lccr = type_from_handle(lc);
-    TClassRef& rccr = type_from_handle(rc);
-
-    if (!lccr.GetClass() || !rccr.GetClass() || scope != (cppyy_scope_t)GLOBAL_HANDLE)
-        return (cppyy_index_t)-1;  // (void*)-1 is in kernel space, so invalid as a method handle
-
-    std::string lcname = lccr->GetName();
-    std::string rcname = rccr->GetName();
-
-    std::string opname = "operator";
-    opname += op;
-
-    for (int idx = 0; idx < (int)g_globalfuncs.size(); ++idx) {
-        TFunction* func = &g_globalfuncs[idx];
-        if (func->GetListOfMethodArgs()->GetSize() != 2)
-            continue;
-
-        if (func->GetName() == opname) {
-            if (lcname == resolve_typedef(((TMethodArg*)func->GetListOfMethodArgs()->At(0))->GetTypeName()) &&
-                rcname == resolve_typedef(((TMethodArg*)func->GetListOfMethodArgs()->At(1))->GetTypeName())) {
-                return (cppyy_index_t)func;
-            }
-        }
-    }
-
-    return (cppyy_index_t)-1;
-}
-
-
-/* method properties -----------------------------------------------------  */
-int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    TMethod* m = (TMethod*)cr->GetListOfMethods()->At(idx);
-    return strcmp(m->GetName(), ((G__ClassInfo*)cr->GetClassInfo())->Name()) == 0;
-}
-
-int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t idx) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    TMethod* m = (TMethod*)cr->GetListOfMethods()->At(idx);
-    return m->Property() & G__BIT_ISSTATIC;
-}
-
-
-/* data member reflection information ------------------------------------- */
-int cppyy_num_datamembers(cppyy_scope_t handle) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass() && cr->GetListOfDataMembers())
-        return cr->GetListOfDataMembers()->GetSize();
-    else if (handle == (cppyy_scope_t)GLOBAL_HANDLE) {
-        TCollection* vars = gROOT->GetListOfGlobals(kTRUE);
-       	if (g_globalvars.size() != (GlobalVars_t::size_type)vars->GetSize()) {
-            g_globalvars.clear();
-	    g_globalvars.reserve(vars->GetSize());
-
-            TIter ivar(vars);
-
-            TGlobal* var = 0;
-            while ((var = (TGlobal*)ivar.Next()))
-                g_globalvars.push_back(*var);
-
-        }
-	return (int)g_globalvars.size();
-    }
-    return 0;
-}
-
-char* cppyy_datamember_name(cppyy_scope_t handle, int datamember_index) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
-        return cppstring_to_cstring(m->GetName());
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    TGlobal& gbl = g_globalvars[datamember_index];
-    return cppstring_to_cstring(gbl.GetName());
-}
-
-char* cppyy_datamember_type(cppyy_scope_t handle, int datamember_index) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass())  {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
-        std::string fullType = m->GetFullTypeName();
-        if ((int)m->GetArrayDim() > 1 || (!m->IsBasic() && m->IsaPointer()))
-            fullType.append("*");
-        else if ((int)m->GetArrayDim() == 1) {
-            std::ostringstream s;
-            s << '[' << m->GetMaxIndex(0) << ']' << std::ends;
-            fullType.append(s.str());
-        }
-        return cppstring_to_cstring(fullType);
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    TGlobal& gbl = g_globalvars[datamember_index];
-    return cppstring_to_cstring(gbl.GetFullTypeName());
-}
-
-ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int datamember_index) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
-        return (ptrdiff_t)m->GetOffsetCint();
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    TGlobal& gbl = g_globalvars[datamember_index];
-    return (ptrdiff_t)gbl.GetAddress();
-}
-
-int cppyy_datamember_index(cppyy_scope_t handle, const char* name) {
-    R__LOCKGUARD2(gCINTMutex);
-
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        // called from updates; add a hard reset as the code itself caches in
-        // Class (TODO: by-pass ROOT/meta)
-        Cppyy_OpenedTClass* c = (Cppyy_OpenedTClass*)cr.GetClass();
-        if (c->fData) {
-            c->fData->Delete();
-            delete c->fData; c->fData = 0;
-            delete c->fAllPubData; c->fAllPubData = 0;
-        }
-        // the following appears dumb, but TClass::GetDataMember() does a linear
-        // search itself, so there is no gain
-        int idm = 0;
-        TDataMember* dm;
-        TIter next(cr->GetListOfDataMembers());
-        while ((dm = (TDataMember*)next())) {
-            if (strcmp(name, dm->GetName()) == 0) {
-                if (dm->Property() & G__BIT_ISPUBLIC)
-                    return idm;
-                return -1;
-            }
-            ++idm;
-        }
-    } else if (handle == (cppyy_type_t)GLOBAL_HANDLE) {
-        TGlobal* gbl = (TGlobal*)gROOT->GetListOfGlobals(kTRUE)->FindObject(name);
-        if (!gbl)
-            return -1;
-        int idx = g_globalvars.size();
-        g_globalvars.push_back(*gbl);
-        return idx;
-    }
-    return -1;
-}
-
-
-/* data member properties ------------------------------------------------  */
-int cppyy_is_publicdata(cppyy_scope_t handle, int datamember_index) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
-        return m->Property() & G__BIT_ISPUBLIC;
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    return 1;  // global data is always public
-}
-
-int cppyy_is_staticdata(cppyy_scope_t handle, int datamember_index) {
-    R__LOCKGUARD2(gCINTMutex);
-    TClassRef& cr = type_from_handle(handle);
-    if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
-        return m->Property() & G__BIT_ISSTATIC;
-    }
-    assert(handle == (cppyy_type_t)GLOBAL_HANDLE);
-    return 1;  // global data is always static
-}
-
-
-/* misc helpers ----------------------------------------------------------- */
-long long cppyy_strtoll(const char* str) {
-    return strtoll(str, NULL, 0);
-}
-
-extern "C" unsigned long long cppyy_strtoull(const char* str) {
-    return strtoull(str, NULL, 0);
-}
-
-void cppyy_free(void* ptr) {
-    free(ptr);
-}
-
-cppyy_object_t cppyy_charp2stdstring(const char* str) {
-    return (cppyy_object_t)new std::string(str);
-}
-
-cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
-    return (cppyy_object_t)new std::string(*(std::string*)ptr);
-}
-
-
-void* cppyy_load_dictionary(const char* lib_name) {
-    R__LOCKGUARD2(gCINTMutex);
-    if (0 <= gSystem->Load(lib_name))
-        return (void*)1;
-    return (void*)0;
-}
-
-
-/* pythonization helpers -------------------------------------------------- */
-typedef double (*tfn_callback)(double*, double*);
-
-cppyy_object_t cppyy_create_tf1(const char* funcname, unsigned long address,
-        double xmin, double xmax, int npar) {
-    return (cppyy_object_t)new TF1(funcname, (tfn_callback)address, xmin, xmax, npar);
-}
-
-cppyy_object_t cppyy_ttree_Branch(void* vtree, const char* branchname, const char* classname,
-        void* addobj, int bufsize, int splitlevel) {
-    // this little song-and-dance is to by-pass the handwritten Branch methods
-    TBranch* b = ((TTree*)vtree)->Bronch(branchname, classname, (void*)&addobj, bufsize, splitlevel);
-    if (b) b->SetObject(addobj);
-    return (cppyy_object_t)b;
-}
-
-long long cppyy_ttree_GetEntry(void* vtree, long long entry) {
-    return (long long)((TTree*)vtree)->GetEntry((Long64_t)entry);
-}
-
-cppyy_object_t cppyy_charp2TString(const char* str) {
-    return (cppyy_object_t)new TString(str);
-}
-
-cppyy_object_t cppyy_TString2TString(cppyy_object_t ptr) {
-    return (cppyy_object_t)new TString(*(TString*)ptr);
-}
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
deleted file mode 100644
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ /dev/null
@@ -1,627 +0,0 @@
-#include "cppyy.h"
-#include "reflexcwrapper.h"
-
-#include "Reflex/Kernel.h"
-#include "Reflex/Type.h"
-#include "Reflex/Base.h"
-#include "Reflex/Member.h"
-#include "Reflex/Object.h"
-#include "Reflex/Builder/TypeBuilder.h"
-#include "Reflex/PropertyList.h"
-#include "Reflex/TypeTemplate.h"
-
-#define private public
-#include "Reflex/PluginService.h"
-#undef private
-
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-#include <assert.h>
-#include <stdlib.h>
-
-
-/* local helpers ---------------------------------------------------------- */
-static inline char* cppstring_to_cstring(const std::string& name) {
-    char* name_char = (char*)malloc(name.size() + 1);
-    strcpy(name_char, name.c_str());
-    return name_char;
-}
-
-static inline Reflex::Scope scope_from_handle(cppyy_type_t handle) {
-    return Reflex::Scope((Reflex::ScopeName*)handle);
-}
-
-static inline Reflex::Type type_from_handle(cppyy_type_t handle) {
-    return Reflex::Scope((Reflex::ScopeName*)handle);
-}
-
-static inline std::vector<void*> build_args(int nargs, void* args) {
-    std::vector<void*> arguments;
-    arguments.reserve(nargs);
-    for (int i = 0; i < nargs; ++i) {
-	char tc = ((CPPYY_G__value*)args)[i].type;
-        if (tc != 'a' && tc != 'o')
-            arguments.push_back(&((CPPYY_G__value*)args)[i]);
-        else
-            arguments.push_back((void*)(*(long*)&((CPPYY_G__value*)args)[i]));
-    }
-    return arguments;
-}
-
-
-/* name to opaque C++ scope representation -------------------------------- */
-int cppyy_num_scopes(cppyy_scope_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    return s.SubScopeSize();
-}
-
-char* cppyy_scope_name(cppyy_scope_t handle, int iscope) {
-    Reflex::Scope s = scope_from_handle(handle);
-    std::string name = s.SubScopeAt(iscope).Name(Reflex::F);
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_resolve_name(const char* cppitem_name) {
-    Reflex::Scope s = Reflex::Scope::ByName(cppitem_name);
-    if (s.IsEnum())
-        return cppstring_to_cstring("unsigned int");
-    const std::string& name = s.Name(Reflex::SCOPED|Reflex::QUALIFIED|Reflex::FINAL);
-    if (name.empty())
-        return cppstring_to_cstring(cppitem_name);
-    return cppstring_to_cstring(name);
-}
-
-cppyy_scope_t cppyy_get_scope(const char* scope_name) {
-    Reflex::Scope s = Reflex::Scope::ByName(scope_name);
-    if (!s) Reflex::PluginService::Instance().LoadFactoryLib(scope_name);
-    s = Reflex::Scope::ByName(scope_name);
-    if (s.IsEnum())     // pretend to be builtin by returning 0
-        return (cppyy_type_t)0;
-    return (cppyy_type_t)s.Id();
-}
-
-cppyy_type_t cppyy_get_template(const char* template_name) {
-   Reflex::TypeTemplate tt = Reflex::TypeTemplate::ByName(template_name);
-   return (cppyy_type_t)tt.Id();
-}
-
-cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) {
-    Reflex::Type t = type_from_handle(klass);
-    Reflex::Type tActual = t.DynamicType(Reflex::Object(t, (void*)obj));
-    if (tActual && tActual != t) {
-        // TODO: lookup through name should not be needed (but tActual.Id()
-        // does not return a singular Id for the system :( )
-        return (cppyy_type_t)cppyy_get_scope(tActual.Name().c_str());
-    }
-    return klass;
-}
-
-
-/* memory management ------------------------------------------------------ */
-cppyy_object_t cppyy_allocate(cppyy_type_t handle) {
-    Reflex::Type t = type_from_handle(handle);
-    return (cppyy_object_t)t.Allocate();
-}
-
-void cppyy_deallocate(cppyy_type_t handle, cppyy_object_t instance) {
-    Reflex::Type t = type_from_handle(handle);
-    t.Deallocate((void*)instance);
-}
-
-void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
-    Reflex::Type t = type_from_handle(handle);
-    t.Destruct((void*)self, true);
-}
-
-
-/* method/function dispatching -------------------------------------------- */
-void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    std::vector<void*> arguments = build_args(nargs, args);
-    Reflex::StubFunction stub = (Reflex::StubFunction)method;
-    stub(NULL /* return address */, (void*)self, arguments, NULL /* stub context */);
-}
-
-template<typename T>
-static inline T cppyy_call_T(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    T result;
-    std::vector<void*> arguments = build_args(nargs, args);
-    Reflex::StubFunction stub = (Reflex::StubFunction)method;
-    stub(&result, (void*)self, arguments, NULL /* stub context */);
-    return result;
-}
-
-unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return (unsigned char)cppyy_call_T<bool>(method, self, nargs, args);
-}
-
-char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<char>(method, self, nargs, args);
-}
-
-short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<short>(method, self, nargs, args);
-}
-
-int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<int>(method, self, nargs, args);
-}
-
-long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<long>(method, self, nargs, args);
-}
-
-long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<long long>(method, self, nargs, args);
-}
-
-float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<float>(method, self, nargs, args);
-}
-
-double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return cppyy_call_T<double>(method, self, nargs, args);
-}
-
-void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    return (void*)cppyy_call_T<long>(method, self, nargs, args);
-}
-
-char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
-    std::string* cppresult = (std::string*)malloc(sizeof(std::string));
-    std::vector<void*> arguments = build_args(nargs, args);
-    Reflex::StubFunction stub = (Reflex::StubFunction)method;
-    stub(cppresult, (void*)self, arguments, NULL /* stub context */);
-    char* cstr = cppstring_to_cstring(*cppresult);
-    cppresult->std::string::~string();
-    free((void*)cppresult);        // the stub will have performed a placement-new
-    return cstr;
-}
-
-cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
-    cppyy_object_t self = cppyy_allocate(handle);
-    cppyy_call_v(method, self, nargs, args);
-    return self;
-}
-
-cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args,
-                  cppyy_type_t result_type) {
-    void* result = (void*)cppyy_allocate(result_type);
-    std::vector<void*> arguments = build_args(nargs, args);
-    Reflex::StubFunction stub = (Reflex::StubFunction)method;
-    stub(result, (void*)self, arguments, NULL /* stub context */);
-    return (cppyy_object_t)result;
-}
-
-static cppyy_methptrgetter_t get_methptr_getter(Reflex::Member m) {
-    Reflex::PropertyList plist = m.Properties();
-    if (plist.HasProperty("MethPtrGetter")) {
-        Reflex::Any& value = plist.PropertyValue("MethPtrGetter");
-        return (cppyy_methptrgetter_t)Reflex::any_cast<void*>(value);
-    }
-    return 0;
-}
-
-cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return get_methptr_getter(m);
-}
-
-
-/* handling of function argument buffer ----------------------------------- */
-void* cppyy_allocate_function_args(int nargs) {
-    CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
-    for (int i = 0; i < nargs; ++i)
-        args[i].type = 'l';
-    return (void*)args;
-}
-
-void cppyy_deallocate_function_args(void* args) {
-    free(args);
-}
-
-size_t cppyy_function_arg_sizeof() {
-    return sizeof(CPPYY_G__value);
-}
-
-size_t cppyy_function_arg_typeoffset() {
-    return offsetof(CPPYY_G__value, type);
-}
-
-
-/* scope reflection information ------------------------------------------- */
-int cppyy_is_namespace(cppyy_scope_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    return s.IsNamespace();
-}
-
-int cppyy_is_enum(const char* type_name) {
-    Reflex::Type t = Reflex::Type::ByName(type_name);
-    return t.IsEnum();
-}
-
-
-/* class reflection information ------------------------------------------- */
-char* cppyy_final_name(cppyy_type_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    if (s.IsEnum())
-        return cppstring_to_cstring("unsigned int");
-    std::string name = s.Name(Reflex::FINAL);
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_scoped_final_name(cppyy_type_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    if (s.IsEnum())
-        return cppstring_to_cstring("unsigned int");
-    std::string name = s.Name(Reflex::SCOPED | Reflex::FINAL);
-    return cppstring_to_cstring(name);
-}   
-
-static int cppyy_has_complex_hierarchy(const Reflex::Type& t) {
-    int is_complex = 1;
-    
-    size_t nbases = t.BaseSize();
-    if (1 < nbases)
-        is_complex = 1;
-    else if (nbases == 0)
-        is_complex = 0;
-    else {         // one base class only
-        Reflex::Base b = t.BaseAt(0);
-        if (b.IsVirtual())
-            is_complex = 1;       // TODO: verify; can be complex, need not be.
-        else
-            is_complex = cppyy_has_complex_hierarchy(t.BaseAt(0).ToType());
-    }
-
-    return is_complex;
-}   
-
-int cppyy_has_complex_hierarchy(cppyy_type_t handle) {
-    Reflex::Type t = type_from_handle(handle);
-    return cppyy_has_complex_hierarchy(t);
-}
-
-int cppyy_num_bases(cppyy_type_t handle) {
-    Reflex::Type t = type_from_handle(handle);
-    std::string name = t.Name(Reflex::FINAL|Reflex::SCOPED);
-    if (5 < name.size() && name.substr(0, 5) == "std::") {
-        // special case: STL base classes are usually unnecessary,
-        // so either build all (i.e. if available) or none
-        for (int i=0; i < (int)t.BaseSize(); ++i)
-            if (!t.BaseAt(i)) return 0;
-    }
-    return t.BaseSize();
-}
-
-char* cppyy_base_name(cppyy_type_t handle, int base_index) {
-    Reflex::Type t = type_from_handle(handle);
-    Reflex::Base b = t.BaseAt(base_index);
-    std::string name = b.Name(Reflex::FINAL|Reflex::SCOPED);
-    return cppstring_to_cstring(name);
-}
-
-int cppyy_is_subtype(cppyy_type_t derived_handle, cppyy_type_t base_handle) {
-    Reflex::Type derived_type = type_from_handle(derived_handle);
-    Reflex::Type base_type = type_from_handle(base_handle);
-    return (int)derived_type.HasBase(base_type);
-}
-
-ptrdiff_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
-                       cppyy_object_t address, int direction) {
-    Reflex::Type derived_type = type_from_handle(derived_handle);
-    Reflex::Type base_type = type_from_handle(base_handle);
-
-    // when dealing with virtual inheritance the only (reasonably) well-defined info is
-    // in a Reflex internal base table, that contains all offsets within the hierarchy
-    Reflex::Member getbases = derived_type.FunctionMemberByName(
-           "__getBasesTable", Reflex::Type(), 0, Reflex::INHERITEDMEMBERS_NO, Reflex::DELAYEDLOAD_OFF);
-    if (getbases) {
-        typedef std::vector<std::pair<Reflex::Base, int> > Bases_t;
-        Bases_t* bases;
-        Reflex::Object bases_holder(Reflex::Type::ByTypeInfo(typeid(Bases_t)), &bases);
-        getbases.Invoke(&bases_holder);
-
-        // if direction is down-cast, perform the cast in C++ first in order to ensure
-        // we have a derived object for accessing internal offset pointers
-        if (direction < 0) {
-           Reflex::Object o(base_type, (void*)address);
-           address = (cppyy_object_t)o.CastObject(derived_type).Address();
-        }
-
-        for (Bases_t::iterator ibase = bases->begin(); ibase != bases->end(); ++ibase) {
-            if (ibase->first.ToType() == base_type) {
-                long offset = (long)ibase->first.Offset((void*)address);
-                if (direction < 0)
-                    return (ptrdiff_t) -offset;  // note negative; rolls over
-                return (ptrdiff_t)offset;
-            }
-        }
-
-        // contrary to typical invoke()s, the result of the internal getbases function
-        // is a pointer to a function static, so no delete
-    }
-
-    return 0;
-}
-
-
-/* method/function reflection information --------------------------------- */
-int cppyy_num_methods(cppyy_scope_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    return s.FunctionMemberSize();
-}
-
-cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth) {
-    return (cppyy_index_t)imeth;
-}
-
-cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t handle, const char* name) {
-    std::vector<cppyy_index_t> result;
-    Reflex::Scope s = scope_from_handle(handle);
-    // the following appears dumb, but the internal storage for Reflex is an
-    // unsorted std::vector anyway, so there's no gain to be had in using the
-    // Scope::FunctionMemberByName() function
-    int num_meth = s.FunctionMemberSize();
-    for (int imeth = 0; imeth < num_meth; ++imeth) {
-        Reflex::Member m = s.FunctionMemberAt(imeth);
-        if (m.Name() == name) {
-            if (m.IsPublic())
-                result.push_back((cppyy_index_t)imeth);
-        }
-    }
-    if (result.empty())
-        return (cppyy_index_t*)0;
-    cppyy_index_t* llresult = (cppyy_index_t*)malloc(sizeof(cppyy_index_t)*(result.size()+1));
-    for (int i = 0; i < (int)result.size(); ++i) llresult[i] = result[i];
-    llresult[result.size()] = -1;
-    return llresult;
-}
-
-char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    std::string name;
-    if (m.IsConstructor())
-        name = s.Name(Reflex::FINAL);   // to get proper name for templates
-    else if (m.IsTemplateInstance()) {
-        name = m.Name();
-        std::string::size_type pos = name.find('<');
-        name = name.substr(0, pos);     // strip template argument portion for overload
-    } else
-        name = m.Name();
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    if (m.IsConstructor())
-        return cppstring_to_cstring("constructor");
-    Reflex::Type rt = m.TypeOf().ReturnType();
-    std::string name = rt.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED);
-    return cppstring_to_cstring(name);
-}
-
-int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return m.FunctionParameterSize();
-}
-
-int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return m.FunctionParameterSize(true);
-}
-
-char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    Reflex::Type at = m.TypeOf().FunctionParameterAt(arg_index);
-    std::string name = at.Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED);
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_method_arg_default(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    std::string dflt = m.FunctionParameterDefaultAt(arg_index);
-    return cppstring_to_cstring(dflt);
-}
-
-char* cppyy_method_signature(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    Reflex::Type mt = m.TypeOf();
-    std::ostringstream sig;
-    if (!m.IsConstructor())
-        sig << mt.ReturnType().Name() << " ";
-    sig << s.Name(Reflex::SCOPED) << "::" << m.Name() << "(";
-    int nArgs = m.FunctionParameterSize();
-    for (int iarg = 0; iarg < nArgs; ++iarg) {
-        sig << mt.FunctionParameterAt(iarg).Name(Reflex::SCOPED|Reflex::QUALIFIED);
-        if (iarg != nArgs-1)
-            sig << ", ";
-    }
-    sig << ")" << std::ends;
-    return cppstring_to_cstring(sig.str());
-}
-
-
-int cppyy_method_is_template(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return m.IsTemplateInstance();
-}
-
-int cppyy_method_num_template_args(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    assert(m.IsTemplateInstance());
-    return m.TemplateArgumentSize();
-}
-
-char* cppyy_method_template_arg_name(
-        cppyy_scope_t handle, cppyy_index_t method_index, cppyy_index_t iarg) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    assert(m.IsTemplateInstance());
-    return cppstring_to_cstring(
-       m.TemplateArgumentAt(iarg).Name(Reflex::SCOPED|Reflex::QUALIFIED));
-}
-
-
-cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return (cppyy_method_t)m.Stubfunction();
-}
-
-cppyy_index_t cppyy_get_global_operator(cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op) {
-    Reflex::Type lct = type_from_handle(lc);
-    Reflex::Type rct = type_from_handle(rc);
-    Reflex::Scope nss = scope_from_handle(scope);
-
-    if (!lct || !rct || !nss) 
-        return (cppyy_index_t)-1;  // (void*)-1 is in kernel space, so invalid as a method handle
-
-    std::string lcname = lct.Name(Reflex::SCOPED|Reflex::FINAL);
-    std::string rcname = rct.Name(Reflex::SCOPED|Reflex::FINAL);
-
-    std::string opname = "operator";
-    opname += op;
-
-    for (int idx = 0; idx < (int)nss.FunctionMemberSize(); ++idx) {
-        Reflex::Member m = nss.FunctionMemberAt(idx);
-        if (m.FunctionParameterSize() != 2)
-            continue;
-
-        if (m.Name() == opname) {
-            Reflex::Type mt = m.TypeOf();
-            if (lcname == mt.FunctionParameterAt(0).Name(Reflex::SCOPED|Reflex::FINAL) &&
-                rcname == mt.FunctionParameterAt(1).Name(Reflex::SCOPED|Reflex::FINAL)) {
-                return (cppyy_index_t)idx;
-            }
-        }
-    }
-
-    return (cppyy_index_t)-1;  
-}
-
-
-/* method properties -----------------------------------------------------  */
-int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return m.IsConstructor();
-}
-
-int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.FunctionMemberAt(method_index);
-    return m.IsStatic();
-}
-
-
-/* data member reflection information ------------------------------------- */
-int cppyy_num_datamembers(cppyy_scope_t handle) {
-    Reflex::Scope s = scope_from_handle(handle);
-    // fix enum representation by adding them to the containing scope as per C++
-    // TODO: this (relatively harmlessly) dupes data members when updating in the
-    //       case s is a namespace
-    for (int isub = 0; isub < (int)s.ScopeSize(); ++isub) {
-        Reflex::Scope sub = s.SubScopeAt(isub);
-        if (sub.IsEnum()) {
-            for (int idata = 0;  idata < (int)sub.DataMemberSize(); ++idata) {
-                Reflex::Member m = sub.DataMemberAt(idata);
-                s.AddDataMember(m.Name().c_str(), sub, 0,
-                                Reflex::PUBLIC|Reflex::STATIC|Reflex::ARTIFICIAL,
-                                (char*)m.Offset());
-            }
-        }
-    }
-    return s.DataMemberSize();
-}
-
-char* cppyy_datamember_name(cppyy_scope_t handle, int datamember_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(datamember_index);
-    std::string name = m.Name();
-    return cppstring_to_cstring(name);
-}
-
-char* cppyy_datamember_type(cppyy_scope_t handle, int datamember_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(datamember_index);
-    std::string name = m.TypeOf().Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED);
-    return cppstring_to_cstring(name);
-}
-
-ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int datamember_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(datamember_index);
-    if (m.IsArtificial() && m.TypeOf().IsEnum())
-        return (ptrdiff_t)&m.InterpreterOffset();
-    return (ptrdiff_t)m.Offset();
-}
-
-int cppyy_datamember_index(cppyy_scope_t handle, const char* name) {
-    Reflex::Scope s = scope_from_handle(handle);
-    // the following appears dumb, but the internal storage for Reflex is an
-    // unsorted std::vector anyway, so there's no gain to be had in using the
-    // Scope::DataMemberByName() function (which returns Member, not an index)
-    int num_dm = cppyy_num_datamembers(handle);
-    for (int idm = 0; idm < num_dm; ++idm) {
-        Reflex::Member m = s.DataMemberAt(idm);
-        if (m.Name() == name || m.Name(Reflex::FINAL) == name) {
-            if (m.IsPublic())
-                return idm;
-            return -1;
-        }
-    }
-    return -1;
-}
-
-
-/* data member properties ------------------------------------------------  */
-int cppyy_is_publicdata(cppyy_scope_t handle, int datamember_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(datamember_index);
-    return m.IsPublic();
-}
-
-int cppyy_is_staticdata(cppyy_scope_t handle, int datamember_index) {
-    Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(datamember_index);
-    return m.IsStatic();
-}
-
-
-/* misc helpers ----------------------------------------------------------- */
-long long cppyy_strtoll(const char* str) {
-    return strtoll(str, NULL, 0);
-}
-
-extern "C" unsigned long long cppyy_strtoull(const char* str) {
-    return strtoull(str, NULL, 0);
-}
-
-void cppyy_free(void* ptr) {
-    free(ptr);
-}
-
-cppyy_object_t cppyy_charp2stdstring(const char* str) {
-    void* arena = new char[sizeof(std::string)];
-    new (arena) std::string(str);
-    return (cppyy_object_t)arena;
-}
-
-cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
-    void* arena = new char[sizeof(std::string)];
-    new (arena) std::string(*(std::string*)ptr);
-    return (cppyy_object_t)arena;
-}

From pypy.commits at gmail.com  Tue Nov 22 20:01:01 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 17:01:01 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: cleanup of data member
 representation
Message-ID: <5834ea4d.471ec20a.76ac6.342d@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88559:29380e1248bf
Date: 2016-11-22 16:30 -0800
http://bitbucket.org/pypy/pypy/changeset/29380e1248bf/

Log:	cleanup of data member representation

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -653,8 +653,8 @@
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
-    def get_returntype(self):
-        return self.space.wrap(self.converter.name)
+    def is_static(self):
+        return self.space.w_False
 
     def _get_offset(self, cppinstance):
         if cppinstance:
@@ -683,13 +683,16 @@
 
 W_CPPDataMember.typedef = TypeDef(
     'CPPDataMember',
-    get_returntype = interp2app(W_CPPDataMember.get_returntype),
+    is_static = interp2app(W_CPPDataMember.is_static),
     __get__ = interp2app(W_CPPDataMember.get),
     __set__ = interp2app(W_CPPDataMember.set),
 )
 W_CPPDataMember.typedef.acceptable_as_base_class = False
 
 class W_CPPStaticData(W_CPPDataMember):
+    def is_static(self):
+        return self.space.w_True
+
     @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         return self.offset
@@ -703,7 +706,7 @@
 
 W_CPPStaticData.typedef = TypeDef(
     'CPPStaticData',
-    get_returntype = interp2app(W_CPPStaticData.get_returntype),
+    is_static = interp2app(W_CPPStaticData.is_static),
     __get__ = interp2app(W_CPPStaticData.get),
     __set__ = interp2app(W_CPPStaticData.set),
 )
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -184,11 +184,10 @@
 
     # add all data members to the dictionary of the class to be created, and
     # static ones also to the meta class (needed for property setters)
-    import cppyy # for _is_static (FIXME)
     for name in cppclass.get_datamember_names():
         cppdm = cppclass.get_datamember(name)
         d_class[name] = cppdm
-        if cppyy._is_static(cppdm):     # TODO: make this a method of cppdm
+        if cppdm.is_static():
             d_meta[name] = cppdm
 
     # create a meta class to allow properties (for static data write access)
@@ -254,7 +253,7 @@
         try:
             cppdm = scope._cpp_proxy.get_datamember(name)
             setattr(scope, name, cppdm)
-            if cppyy._is_static(cppdm): # TODO: make this a method of cppdm
+            if cppdm.is_static():
                 setattr(scope.__class__, name, cppdm)
             pycppitem = getattr(scope, name)      # gets actual property value
         except AttributeError:

From pypy.commits at gmail.com  Tue Nov 22 20:01:07 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 17:01:07 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: restore dummy backend for some
 moderate testing w/o Cling
Message-ID: <5834ea53.113cc20a.8d5cb.29bd@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88562:863b481929a7
Date: 2016-11-22 16:53 -0800
http://bitbucket.org/pypy/pypy/changeset/863b481929a7/

Log:	restore dummy backend for some moderate testing w/o Cling

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -9,8 +9,8 @@
 # the selection of the desired backend (default is Reflex).
 
 # choose C-API access method:
-#from pypy.module.cppyy.capi.loadable_capi import *
-from pypy.module.cppyy.capi.builtin_capi import *
+from pypy.module.cppyy.capi.loadable_capi import *
+#from pypy.module.cppyy.capi.builtin_capi import *
 
 from pypy.module.cppyy.capi.capi_types import C_OBJECT,\
     C_NULL_TYPE, C_NULL_OBJECT
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -725,7 +725,7 @@
     return result;
 }
 
-char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
+char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* /* length */) {
     char* result = 0;
     const long idx = (long)method;
     if (idx == s_methods["static_example01::staticStrcpy_cchar*"]) {
@@ -759,8 +759,8 @@
     return (cppyy_object_t)result;
 }
 
-cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
-    return (cppyy_methptrgetter_t)0;
+cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) {
+    return (cppyy_funcaddr_t)0;
 }
 
 
@@ -790,7 +790,11 @@
 /* scope reflection information ------------------------------------------- */
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
-}   
+}
+
+int cppyy_is_abstract(cppyy_type_t /* type) */) {
+    return 0;
+}
 
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
@@ -942,9 +946,9 @@
     free(ptr);
 }
 
-cppyy_object_t cppyy_charp2stdstring(const char* str) {
-    void* arena = new char[sizeof(std::string)];
-    new (arena) std::string(str);
+cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz) {
+    void* arena = new char[sz];
+    new (arena) std::string(str, sz);
     return (cppyy_object_t)arena;
 }
 

From pypy.commits at gmail.com  Tue Nov 22 20:01:05 2016
From: pypy.commits at gmail.com (wlav)
Date: Tue, 22 Nov 2016 17:01:05 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: cleanup
Message-ID: <5834ea51.a285c20a.08c8.32a7@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88561:75adba4ebe79
Date: 2016-11-22 16:36 -0800
http://bitbucket.org/pypy/pypy/changeset/75adba4ebe79/

Log:	cleanup

diff --git a/pypy/module/cppyy/__init__.py b/pypy/module/cppyy/__init__.py
--- a/pypy/module/cppyy/__init__.py
+++ b/pypy/module/cppyy/__init__.py
@@ -14,7 +14,6 @@
         '_set_class_generator'   : 'interp_cppyy.set_class_generator',
         '_set_function_generator': 'interp_cppyy.set_function_generator',
         '_register_class'        : 'interp_cppyy.register_class',
-        '_is_static'             : 'interp_cppyy.is_static',
         '_get_nullptr'           : 'interp_cppyy.get_nullptr',
         'CPPInstanceBase'        : 'interp_cppyy.W_CPPInstance',
         'addressof'              : 'interp_cppyy.addressof',

From pypy.commits at gmail.com  Tue Nov 22 22:24:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 19:24:12 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: fix some tests
Message-ID: <58350bdc.4673c20a.e67f3.4b4d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88563:483405a0e443
Date: 2016-11-23 02:20 +0000
http://bitbucket.org/pypy/pypy/changeset/483405a0e443/

Log:	fix some tests

diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py
--- a/rpython/rtyper/test/test_rbuiltin.py
+++ b/rpython/rtyper/test/test_rbuiltin.py
@@ -132,17 +132,11 @@
         assert self.interpret(fn, (1.9, 2.)) == 1.9
         assert self.interpret(fn, (1.5, -1.4)) == -1.4
 
-    def test_float_int_min(self):
-        def fn(i, j):
-            return min(i, j)
-        assert self.interpret(fn, (1.9, 2)) == 1.9
-        assert self.interpret(fn, (1.5, -1)) == -1
-
     def test_float_max(self):
         def fn(i, j):
             return max(i,j)
         assert self.interpret(fn, (1.0, 2.)) == 2
-        assert self.interpret(fn, (1.1, -1)) == 1.1
+        assert self.interpret(fn, (1.1, -1.)) == 1.1
 
     def test_builtin_math_floor(self):
         def fn(f):
@@ -165,18 +159,18 @@
     def test_builtin_math_frexp(self):
         def fn(f):
             return math.frexp(f)
-        for x in (.5, 1, 1.5, 10/3.0):
+        for x in (.5, 1., 1.5, 10/3.0):
             for y in (1, -1):
                 res = self.interpret(fn, [x*y])
                 mantissa, exponent = math.frexp(x*y)
-                assert (self.float_eq(res.item0, mantissa) and
-                        self.float_eq(res.item1, exponent))
+                assert self.float_eq(res.item0, mantissa)
+                assert res.item1 == exponent
 
     def test_builtin_math_ldexp(self):
         def fn(a, b):
             return math.ldexp(a, b)
-        assert self.interpret(fn, [1, 2]) == 4
-        self.interpret_raises(OverflowError, fn, [1, 100000])
+        assert self.interpret(fn, [1., 2]) == 4
+        self.interpret_raises(OverflowError, fn, [1., 100000])
 
     def test_builtin_math_modf(self):
         def fn(f):
diff --git a/rpython/rtyper/test/test_rfloat.py b/rpython/rtyper/test/test_rfloat.py
--- a/rpython/rtyper/test/test_rfloat.py
+++ b/rpython/rtyper/test/test_rfloat.py
@@ -150,9 +150,9 @@
         from rpython.rlib import rfloat
         def fn(x, y):
             return rfloat.copysign(x, y)
-        assert self.interpret(fn, [42, -1]) == -42
-        assert self.interpret(fn, [42, -0.0]) == -42
-        assert self.interpret(fn, [42, 0.0]) == 42
+        assert self.interpret(fn, [42., -1.]) == -42.
+        assert self.interpret(fn, [42., -0.]) == -42.
+        assert self.interpret(fn, [42., 0.]) == 42.
 
     def test_rstring_to_float(self):
         from rpython.rlib.rfloat import rstring_to_float
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1038,7 +1038,7 @@
         fr3 = Freezing()
         fr1.value = 10
         fr2.value = 5
-        fr3.value = 2.5
+        fr3.value = 3
         def pick12(i):
             if i > 0:
                 return fr1
@@ -1056,8 +1056,8 @@
         for i in [0, 5, 10]:
             res = self.interpret(f, [i])
             item0, item1 = self.ll_unpack_tuple(res, 2)
-            assert type(item0) is int   # precise
-            assert type(item1) in (float, int)  # we get int on JS
+            assert type(item0) is int
+            assert type(item1) is int
             assert item0 == f(i)[0]
             assert item1 == f(i)[1]
 
@@ -1068,7 +1068,7 @@
         class fr3(base): pass
         fr1.value = 10
         fr2.value = 5
-        fr3.value = 2.5
+        fr3.value = 3
         def pick12(i):
             if i > 0:
                 return fr1
@@ -1086,8 +1086,8 @@
         for i in [0, 5, 10]:
             res = self.interpret(f, [i])
             item0, item1 = self.ll_unpack_tuple(res, 2)
-            assert type(item0) is int   # precise
-            assert type(item1) in (float, int)  # we get int on JS
+            assert type(item0) is int
+            assert type(item1) is int
             assert item0 == f(i)[0]
             assert item1 == f(i)[1]
 

From pypy.commits at gmail.com  Tue Nov 22 22:24:15 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 19:24:15 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: fixes
Message-ID: <58350bdf.e7b1c20a.de63f.3b3e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88565:dc76b2116960
Date: 2016-11-23 03:23 +0000
http://bitbucket.org/pypy/pypy/changeset/dc76b2116960/

Log:	fixes

diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py
--- a/rpython/jit/metainterp/jitprof.py
+++ b/rpython/jit/metainterp/jitprof.py
@@ -52,8 +52,8 @@
 class Profiler(BaseProfiler):
     initialized = False
     timer = staticmethod(time.time)
-    starttime = 0
-    t1 = 0
+    starttime = 0.
+    t1 = 0.
     times = None
     counters = None
     calls = 0
@@ -63,7 +63,7 @@
     def start(self):
         self.starttime = self.timer()
         self.t1 = self.starttime
-        self.times = [0, 0]
+        self.times = [0., 0.]
         self.counters = [0] * (Counters.ncounters - _CPU_LINES)
         self.calls = 0
         self.current = []
diff --git a/rpython/jit/metainterp/test/test_vector.py b/rpython/jit/metainterp/test/test_vector.py
--- a/rpython/jit/metainterp/test/test_vector.py
+++ b/rpython/jit/metainterp/test/test_vector.py
@@ -478,7 +478,7 @@
             va = lltype.malloc(T, d, flavor='raw', zero=True)
             for i in range(d):
                 va[i] = value
-            r = 0
+            r = 0.
             i = 0
             k = d + 2
             # in this case a guard k <= d is inserted which fails right away!
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -68,7 +68,7 @@
             exp = r_ulonglong(0x7ff) << 52
             sign = r_ulonglong(sign) << 63
             if MANT_DIG < 53:
-                mant = r_ulonglong(mant) << (53 - MANT_DIG) 
+                mant = r_ulonglong(mant) << (53 - MANT_DIG)
             if mant == 0:
                 result = rfloat.NAN
             else:
@@ -77,11 +77,11 @@
             return result
     elif exp == 0:
         # subnormal or zero
-        result = math.ldexp(mant, MIN_EXP - MANT_DIG)
+        result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
     else:
         # normal: add implicit one value
         mant += one << MANT_DIG - 1
-        result = math.ldexp(mant, exp + MIN_EXP - MANT_DIG - 1)
+        result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
     return -result if sign else result
 
 def float_unpack80(QQ, size):
diff --git a/rpython/rlib/test/test_longlong2float.py b/rpython/rlib/test/test_longlong2float.py
--- a/rpython/rlib/test/test_longlong2float.py
+++ b/rpython/rlib/test/test_longlong2float.py
@@ -41,7 +41,7 @@
             ll = float2longlong(f1)
             return longlong2float(ll)
         except Exception:
-            return 500
+            return 500.
 
     for x in enum_floats():
         res = interpret(f, [x])

From pypy.commits at gmail.com  Tue Nov 22 22:24:13 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 19:24:13 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: fix gc.min_heap_size
Message-ID: <58350bdd.d39a1c0a.82b0d.1588@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88564:fede6b49acc1
Date: 2016-11-23 02:55 +0000
http://bitbucket.org/pypy/pypy/changeset/fede6b49acc1/

Log:	fix gc.min_heap_size

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -465,7 +465,7 @@
                 self.min_heap_size = float(min_heap_size)
             else:
                 # defaults to 8 times the nursery
-                self.min_heap_size = newsize * 8
+                self.min_heap_size = float(newsize) * 8
             #
             max_heap_size = env.read_uint_from_env('PYPY_GC_MAX')
             if max_heap_size > 0:
@@ -798,7 +798,7 @@
                 # nursery. "Next area" in this case is the space between the
                 # pinned object in front of nusery_top and the pinned object
                 # after that. Graphically explained:
-                # 
+                #
                 #     |- allocating totalsize failed in this area
                 #     |     |- nursery_top
                 #     |     |    |- pinned object in front of nursery_top,
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -385,7 +385,7 @@
                 self.min_heap_size = float(min_heap_size)
             else:
                 # defaults to 8 times the nursery
-                self.min_heap_size = newsize * 8
+                self.min_heap_size = float(newsize) * 8
             #
             max_heap_size = env.read_uint_from_env('PYPY_GC_MAX')
             if max_heap_size > 0:

From pypy.commits at gmail.com  Wed Nov 23 00:04:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 21:04:21 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: fix tests
Message-ID: <58352355.849c1c0a.38a6c.26c1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88567:4467e17cbe42
Date: 2016-11-23 05:03 +0000
http://bitbucket.org/pypy/pypy/changeset/4467e17cbe42/

Log:	fix tests

diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -24,7 +24,7 @@
     def test_inheritance2(self):
         def wrap():
             res = snippet.inheritance2()
-            return res == ((-12, -12.0), (3, 12.3))
+            return res == ((-12, -12), (3, 12))
         fn = self.getcompiled(wrap, [])
         assert fn()
 
@@ -84,6 +84,7 @@
         assert fn(4) == 789
         assert fn(5) == 101112
 
+    @py.test.mark.xfail(reason="need a test that doesn't require int-float unions")
     def test_type_conversion(self):
         # obfuscated test case specially for typer.insert_link_conversions()
         def type_conversion(n):
@@ -137,7 +138,7 @@
 
     def test_float_ops(self):
         def f(x):
-            return abs(math.pow(-x, 3) + 1)
+            return abs(math.pow(-x, 3.) + 1)
         fn = self.getcompiled(f, [float])
         assert fn(-4.5) == 92.125
         assert fn(4.5) == 90.125
@@ -145,7 +146,7 @@
     def test_memoryerror(self):
         def g(i):
             return [0] * i
-        
+
         def f(i):
             try:
                 lst = g(i)
@@ -505,7 +506,7 @@
         def func(x, y):
             z = x + y / 2.1 * x
             z = math.fmod(z, 60.0)
-            z = math.pow(z, 2)
+            z = math.pow(z, 2.)
             z = -z
             return int(z)
 
@@ -692,7 +693,7 @@
                 return f(n)
             except StackOverflow:
                 return -42
-        
+
         def f(n):
             if n == 0:
                 return 1
diff --git a/rpython/translator/test/snippet.py b/rpython/translator/test/snippet.py
--- a/rpython/translator/test/snippet.py
+++ b/rpython/translator/test/snippet.py
@@ -371,7 +371,7 @@
     d = D()
     d.stuff = (-12, -12)
     e = E()
-    e.stuff = (3, 12.3)
+    e.stuff = (3, 12)
     return _getstuff(d), _getstuff(e)
 
 class F:

From pypy.commits at gmail.com  Wed Nov 23 00:04:19 2016
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 22 Nov 2016 21:04:19 -0800 (PST)
Subject: [pypy-commit] pypy fix-broken-types: fix more tests
Message-ID: <58352353.88711c0a.25ac.26c2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: fix-broken-types
Changeset: r88566:186e0cdffeb1
Date: 2016-11-23 04:29 +0000
http://bitbucket.org/pypy/pypy/changeset/186e0cdffeb1/

Log:	fix more tests

diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -502,13 +502,6 @@
         return a + b
     assert f._annenforceargs_ == (None, int, None)
 
-def test_enforceargs_int_float_promotion():
-    @enforceargs(float)
-    def f(x):
-        return x
-    # in RPython there is an implicit int->float promotion
-    assert f(42) == 42
-
 def test_enforceargs_None_string():
     @enforceargs(str, unicode)
     def f(a, b):
@@ -540,7 +533,7 @@
     @enforceargs(int, float)
     def f(a, b):
         return a, b
-    graph = getgraph(f, [int, int])
+    graph = getgraph(f, [int, float])
     TYPES = [v.concretetype for v in graph.getargs()]
     assert TYPES == [lltype.Signed, lltype.Float]
 
@@ -623,7 +616,7 @@
 def test_iterkeys_with_hash():
     def f(i):
         d = {i + .0: 5, i + .5: 6}
-        total = 0
+        total = 0.
         for k, h in iterkeys_with_hash(d):
             total += k * h
         total -= (i + 0.0) * compute_hash(i + 0.0)
@@ -637,7 +630,7 @@
 def test_iteritems_with_hash():
     def f(i):
         d = {i + .0: 5, i + .5: 6}
-        total = 0
+        total = 0.
         for k, v, h in iteritems_with_hash(d):
             total += k * h * v
         total -= (i + 0.0) * compute_hash(i + 0.0) * 5
diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py
--- a/rpython/rlib/test/test_signature.py
+++ b/rpython/rlib/test/test_signature.py
@@ -257,7 +257,7 @@
         return x + y
     @signature(types.int(), returns=types.float())
     def g(x):
-        return f(x, x)
+        return f(x + 0.5, x)
     sig = getsig(g)
     assert sig == [model.SomeInteger(), model.SomeFloat()]
 
@@ -267,11 +267,12 @@
     sig = getsig(g)
     assert sig == [model.SomeFloat(), model.SomeFloat()]
 
-    @signature(types.str(), returns=types.int())
+    @signature(types.str(), returns=types.float())
     def cannot_add_string(x):
         return f(x, 2)
-    exc = py.test.raises(model.AnnotatorError, annotate_at, cannot_add_string).value
-    assert 'Blocked block' in str(exc)
+    with py.test.raises(model.AnnotatorError) as excinfo:
+        annotate_at(cannot_add_string)
+    assert 'Blocked block' in str(excinfo.value)
 
 def test_return_any():
     @signature(types.int(), returns=types.any())
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -549,7 +549,7 @@
     1, sys.maxint, 1.5, 'a', 'abc', u'abc', None, [],
     lambda: None,
     {1.23: 'abc'},
-    (1, 'x', [2, 3.],),
+    (1, 'x', [2, 3],),
     Frozen(),])
 def test_typeOf_const(x):
     a = RPythonAnnotator()

From pypy.commits at gmail.com  Wed Nov 23 03:16:41 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 00:16:41 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <58355069.44e61c0a.17aa8.68de@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r822:2ac21af4d38d
Date: 2016-11-23 09:16 +0100
http://bitbucket.org/pypy/pypy.org/changeset/2ac21af4d38d/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66265 of $105000 (63.1%)
+   $66267 of $105000 (63.1%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $3567 left
+    we have $2128 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $58990 of $80000 (73.7%)
+   $59000 of $80000 (73.7%)
    <div id="progressbar">
    </div>
 
@@ -29,7 +29,7 @@
   
   <!--
   Current status:
-    we have $48212 left
+    we have $48221 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Wed Nov 23 04:09:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 01:09:22 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: in-progress
Message-ID: <58355cc2.e337c20a.a96e4.a74c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88568:2353e0a7c50b
Date: 2016-11-23 10:08 +0100
http://bitbucket.org/pypy/pypy/changeset/2353e0a7c50b/

Log:	in-progress

diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1569,7 +1569,7 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
+    def _rewrite_op_cond_call(self, op, rewritten_opname):
         have_floats = False
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
@@ -1580,13 +1580,18 @@
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
-        op1 = self.rewrite_call(op, 'conditional_call',
+        op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
                                 calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_conditional_call(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call')
+    def rewrite_op_jit_conditional_call_value(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call_value')
+
     def handle_jit_marker__jit_merge_point(self, op, jitdriver):
         assert self.portal_jd is not None, (
             "'jit_merge_point' in non-portal graph!")
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1199,13 +1199,27 @@
     def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr):
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
-    # conditional calls - note that they cannot return stuff
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
+        # conditional calls - condition is a flag, and they cannot return stuff
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
+    @arguments("cpu", "i", "i", "I", "R", "d")
+    def bhimpl_conditional_call_value_ir_i(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if value == 0:
+            value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
+        return value
+
+    @arguments("cpu", "r", "i", "I", "R", "d")
+    def bhimpl_conditional_call_value_ir_r(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if not value:
+            value = cpu.bh_call_r(func, args_i, args_r, None, calldescr)
+        return value
+
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1059,8 +1059,19 @@
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
+        if isinstance(condbox, ConstInt) and condbox.value == 0:
+            return   # so that the heapcache can keep argboxes virtual
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
+    @arguments("box", "box", "boxes2", "descr", "orgpc")
+    def _opimpl_conditional_call_value(self, valuebox, funcbox, argboxes,
+                                       calldescr, pc):
+        return self.do_conditional_call(valuebox, funcbox, argboxes,
+                                        calldescr, pc, is_value=True)
+
+    opimpl_conditional_call_value_ir_i = _opimpl_conditional_call_value
+    opimpl_conditional_call_value_ir_r = _opimpl_conditional_call_value
+
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
@@ -1538,7 +1549,7 @@
                                                             descr=descr)
         if pure and not self.metainterp.last_exc_value and op:
             op = self.metainterp.record_result_of_call_pure(op, argboxes, descr,
-                patch_pos)
+                patch_pos, opnum)
             exc = exc and not isinstance(op, Const)
         if exc:
             if op is not None:
@@ -1712,16 +1723,21 @@
             else:
                 assert False
 
-    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
-        if isinstance(condbox, ConstInt) and condbox.value == 0:
-            return   # so that the heapcache can keep argboxes virtual
+    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc,
+                            is_value=False):
         allboxes = self._build_allboxes(funcbox, argboxes, descr)
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop.COND_CALL, [condbox] + allboxes, descr,
-                                    exc, pure)
+        if not is_value:
+            opnum = rop.COND_CALL
+        else:
+            opnum = OpHelpers.cond_call_value_for_descr(descr)
+        # COND_CALL cannot be pure (=elidable): it has no result.
+        # On the other hand, COND_CALL_VALUE is always calling a pure
+        # function.
+        return self.execute_varargs(opnum, [condbox] + allboxes, descr,
+                                    exc, pure=is_value)
 
     def _do_jit_force_virtual(self, allboxes, descr, pc):
         assert len(allboxes) == 2
@@ -3061,11 +3077,16 @@
         debug_stop("jit-abort-longest-function")
         return max_jdsd, max_key
 
-    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos):
+    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos, opnum):
         """ Patch a CALL into a CALL_PURE.
         """
         resbox_as_const = executor.constant_from_op(op)
-        for argbox in argboxes:
+        is_cond_value = OpHelpers.is_cond_call_value(opnum)
+        if is_cond_value:
+            normargboxes = argboxes[1:]    # ingore the 'value' arg
+        else:
+            normargboxes = argboxes
+        for argbox in normargboxes:
             if not isinstance(argbox, Const):
                 break
         else:
@@ -3077,6 +3098,8 @@
         # be either removed later by optimizeopt or turned back into CALL.
         arg_consts = [executor.constant_from_op(a) for a in argboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
+        if is_cond_value:
+            return op       # but COND_CALL_VALUE remains
         opnum = OpHelpers.call_pure_for_descr(descr)
         self.history.cut(patch_pos)
         newop = self.history.record_nospec(opnum, argboxes, descr)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1151,7 +1151,7 @@
     '_CALL_FIRST',
     'CALL/*d/rfin',
     'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
-    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
+    'COND_CALL_VALUE/*d/ri',  # "return a0 or a1(a2, ..)", a1 elidable
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',
@@ -1274,6 +1274,15 @@
         return rop.CALL_LOOPINVARIANT_N
 
     @staticmethod
+    def cond_call_value_for_descr(descr):
+        tp = descr.get_normalized_result_type()
+        if tp == 'i':
+            return rop.COND_CALL_VALUE_I
+        elif tp == 'r':
+            return rop.COND_CALL_VALUE_R
+        assert False, tp
+
+    @staticmethod
     def getfield_pure_for_descr(descr):
         if descr.is_pointer_field():
             return rop.GETFIELD_GC_PURE_R
@@ -1447,6 +1456,11 @@
                 opnum == rop.CALL_RELEASE_GIL_N)
 
     @staticmethod
+    def is_cond_call_value(opnum):
+        return (opnum == rop.COND_CALL_VALUE_I or
+                opnum == rop.COND_CALL_VALUE_R)
+
+    @staticmethod
     def is_ovf(opnum):
         return rop._OVF_FIRST <= opnum <= rop._OVF_LAST
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4575,3 +4575,15 @@
 
         self.meta_interp(g, [5, 5, 5])
         self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
+
+    def test_conditional_call_value(self):
+        from rpython.rlib.jit import conditional_call_value
+        @elidable
+        def g(j):
+            return j + 5
+        def f(i, j):
+            return conditional_call_value(i, g, j)
+        res = self.interp_operations(f, [-42, 200])
+        assert res == -42
+        res = self.interp_operations(f, [0, 200])
+        assert res == 205
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1246,6 +1246,9 @@
                                                  args_s[1], args_s[2:])
         if self.instance == _jit_conditional_call_value:
             from rpython.annotator import model as annmodel
+            func = args_s[1].const
+            # conditional_call_value(): function must be elidable
+            assert func._elidable_function_
             return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -303,6 +303,7 @@
         mix.finish()
 
     def test_conditional_call_value(self):
+        @elidable
         def g(x, y):
             return x - y + 5
         def f(n, x, y):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -431,6 +431,7 @@
     'jit_record_exact_class'  : LLOp(canrun=True),
     'jit_ffi_save_result':  LLOp(canrun=True),
     'jit_conditional_call': LLOp(),
+    'jit_conditional_call_value': LLOp(),
     'jit_enter_portal_frame': LLOp(canrun=True),
     'jit_leave_portal_frame': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -456,6 +456,9 @@
     def OP_JIT_CONDITIONAL_CALL(self, op):
         return 'abort();  /* jit_conditional_call */'
 
+    def OP_JIT_CONDITIONAL_CALL_VALUE(self, op):
+        return 'abort();  /* jit_conditional_call_value */'
+
     # low-level operations
     def generic_get(self, op, sourceexpr):
         T = self.lltypemap(op.result)

From pypy.commits at gmail.com  Wed Nov 23 04:28:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 01:28:16 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Fix the logic: "not
 value" is not enough, we need "value is None"
Message-ID: <58356130.8b9a1c0a.d3fdb.847e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88570:db9ba07ec696
Date: 2016-11-23 10:27 +0100
http://bitbucket.org/pypy/pypy/changeset/db9ba07ec696/

Log:	Fix the logic: "not value" is not enough, we need "value is None"

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -77,6 +77,8 @@
             l = jit.conditional_call_value(l, f, n)
             return len(l)
 
+        assert main(10) == 0
+        assert main(5) == 1
         assert self.interp_operations(main, [10]) == 0
         assert self.interp_operations(main, [5]) == 1
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1221,20 +1221,27 @@
 def conditional_call_value(value, function, *args):
     """Does the same as:
 
-        if not value:
+        if value == <0 or None>:
             value = function(*args)
         return value
 
     For the JIT.  Allows one branch which doesn't create a bridge,
     typically used for caching.  The function must be @elidable.
     The value and the function's return type must match and cannot
-    be a float.
+    be a float: they must be either regular 'int', or something
+    that turns into a pointer.
     """
     if we_are_jitted():
         return _jit_conditional_call_value(value, function, *args)
     else:
-        if not value:
-            value = function(*args)
+        if isinstance(value, int):
+            if value == 0:
+                value = function(*args)
+                assert isinstance(value, int)
+        else:
+            if value is None:
+                value = function(*args)
+                assert not isinstance(value, int)
         return value
 conditional_call_value._always_inline_ = True
 

From pypy.commits at gmail.com  Wed Nov 23 04:28:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 01:28:14 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Start to adapt tests
 from conditional_call_value_2
Message-ID: <5835612e.c515c20a.cc3bd.c541@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88569:5482941a8917
Date: 2016-11-23 10:22 +0100
http://bitbucket.org/pypy/pypy/changeset/5482941a8917/

Log:	Start to adapt tests from conditional_call_value_2

diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.CallTest):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,6 +271,7 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
+            OpHelpers.is_cond_call_value(opnum) or
             opnum == rop.COND_CALL):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1066,6 +1066,8 @@
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def _opimpl_conditional_call_value(self, valuebox, funcbox, argboxes,
                                        calldescr, pc):
+        if isinstance(valuebox, Const) and valuebox.nonnull():
+            return valuebox
         return self.do_conditional_call(valuebox, funcbox, argboxes,
                                         calldescr, pc, is_value=True)
 
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -2,7 +2,7 @@
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib import jit
 
-class TestCall(LLJitMixin):
+class CallTest(object):
     def test_indirect_call(self):
         @jit.dont_look_inside
         def f1(x):
@@ -52,3 +52,128 @@
 
         assert self.meta_interp(main, [10]) == 42
         self.check_resops(guard_no_exception=0)
+
+    def test_cond_call_i(self):
+        @jit.elidable
+        def f(n):
+            return n * 200
+
+        def main(n, m):
+            return jit.conditional_call_value(n, f, m)
+
+        assert self.interp_operations(main, [0, 10]) == 2000
+        assert self.interp_operations(main, [15, 42]) == 15
+
+    def test_cond_call_r(self):
+        @jit.elidable
+        def f(n):
+            return [n]
+
+        def main(n):
+            if n == 10:
+                l = []
+            else:
+                l = None
+            l = jit.conditional_call_value(l, f, n)
+            return len(l)
+
+        assert self.interp_operations(main, [10]) == 0
+        assert self.interp_operations(main, [5]) == 1
+
+    def test_cond_call_constant_in_pyjitpl(self):
+        def f(a, b):
+            return a + b
+        def main(n):
+            # this is completely constant-folded because the arguments
+            # to f() are constants.
+            return jit.conditional_call_elidable(n, 23, f, 40, 2)
+
+        assert main(12) == 12                            # because 12 != 23
+        assert self.interp_operations(main, [12]) == 12  # because 12 != 23
+        self.check_operations_history(finish=1)   # empty history
+        assert self.interp_operations(main, [23]) == 42  # because 23 == 23
+        self.check_operations_history(finish=1)   # empty history
+
+    def test_cond_call_constant_in_optimizer(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return x - 3
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                v = V(m)
+                n -= jit.conditional_call_elidable(p, -42, externfn, v.value)
+            return n
+        res = self.meta_interp(f, [21, 5, -42])
+        assert res == -1
+        # the COND_CALL_PURE is constant-folded away by optimizeopt.py
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=0,
+                          int_sub=2)
+
+    def test_cond_call_constant_in_optimizer_2(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return 2
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > 14
+                assert p < 16
+                n -= jit.conditional_call_elidable(p, 15, externfn, n)
+            return n
+        res = self.meta_interp(f, [21, 5, 15])
+        assert res == -1
+        # optimizer: the COND_CALL_PURE is turned into a regular
+        # CALL_PURE, which itself becomes CALL
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+                          int_sub=2)
+
+    def test_cond_call_constant_in_optimizer_3(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return 1
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > 14
+                assert p < 16
+                n0 = n
+                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
+                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
+            return n
+        res = self.meta_interp(f, [21, 5, 15])
+        assert res == -1
+        # same as test_cond_call_constant_in_optimizer_2, but the two
+        # intermediate CALL_PUREs are replaced with only one, because
+        # they are called with the same arguments
+        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+                          int_sub=4)
+
+    def test_cond_call_constant_in_optimizer_4(self):
+        class X:
+            def __init__(self, value):
+                self.value = value
+                self.triple = -1
+            def _compute_triple(self):
+                self.triple = self.value * 3
+                return self.triple
+            def get_triple(self):
+                return jit.conditional_call_elidable(self.triple, -1,
+                                                     X._compute_triple, self)
+        def main(n, initvalue):
+            x = X(n)
+            x.triple = initvalue
+            return x.get_triple() + x.get_triple()
+
+        assert self.interp_operations(main, [100, -1]) == 600
+        self.check_operations_history(finish=1)   # empty history
+
+
+class TestCall(LLJitMixin, CallTest):
+    pass
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1247,8 +1247,9 @@
         if self.instance == _jit_conditional_call_value:
             from rpython.annotator import model as annmodel
             func = args_s[1].const
-            # conditional_call_value(): function must be elidable
-            assert func._elidable_function_
+            assert getattr(func, '_elidable_function_', None), (
+                "%r used in jit.conditional_call_value() should be "
+                "@jit.elidable" % (func,))
             return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):

From pypy.commits at gmail.com  Wed Nov 23 04:40:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 01:40:14 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: adapt tests
Message-ID: <583563fe.52301c0a.d82bc.8af5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88571:983b2cd122c4
Date: 2016-11-23 10:39 +0100
http://bitbucket.org/pypy/pypy/changeset/983b2cd122c4/

Log:	adapt tests

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -83,21 +83,24 @@
         assert self.interp_operations(main, [5]) == 1
 
     def test_cond_call_constant_in_pyjitpl(self):
+        @jit.elidable
         def f(a, b):
             return a + b
         def main(n):
             # this is completely constant-folded because the arguments
             # to f() are constants.
-            return jit.conditional_call_elidable(n, 23, f, 40, 2)
+            return jit.conditional_call_value(n, f, 40, 2)
 
-        assert main(12) == 12                            # because 12 != 23
-        assert self.interp_operations(main, [12]) == 12  # because 12 != 23
+        assert main(12) == 12
+        assert main(0) == 42
+        assert self.interp_operations(main, [12]) == 12
         self.check_operations_history(finish=1)   # empty history
-        assert self.interp_operations(main, [23]) == 42  # because 23 == 23
+        assert self.interp_operations(main, [0]) == 42
         self.check_operations_history(finish=1)   # empty history
 
     def test_cond_call_constant_in_optimizer(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
         def externfn(x):
             return x - 3
         class V:
@@ -108,46 +111,48 @@
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
                 myjitdriver.jit_merge_point(n=n, p=p, m=m)
                 v = V(m)
-                n -= jit.conditional_call_elidable(p, -42, externfn, v.value)
+                n -= jit.conditional_call_value(p, externfn, v.value)
             return n
-        res = self.meta_interp(f, [21, 5, -42])
+        res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
-        # the COND_CALL_PURE is constant-folded away by optimizeopt.py
+        # the COND_CALL_VALUE is constant-folded away by optimizeopt.py
         self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=0,
                           int_sub=2)
 
     def test_cond_call_constant_in_optimizer_2(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
         def externfn(x):
             return 2
         def f(n, m, p):
             while n > 0:
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
                 myjitdriver.jit_merge_point(n=n, p=p, m=m)
-                assert p > 14
-                assert p < 16
-                n -= jit.conditional_call_elidable(p, 15, externfn, n)
+                assert p > -1
+                assert p < 1
+                n -= jit.conditional_call_value(p, externfn, n)
             return n
-        res = self.meta_interp(f, [21, 5, 15])
+        res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
-        # optimizer: the COND_CALL_PURE is turned into a regular
-        # CALL_PURE, which itself becomes CALL
+        # optimizer: the COND_CALL_VALUE is turned into a regular
+        # CALL_PURE, which itself becomes a CALL
         self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
                           int_sub=2)
 
     def test_cond_call_constant_in_optimizer_3(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        @jit.elidable
         def externfn(x):
             return 1
         def f(n, m, p):
             while n > 0:
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
                 myjitdriver.jit_merge_point(n=n, p=p, m=m)
-                assert p > 14
-                assert p < 16
+                assert p > -1
+                assert p < 1
                 n0 = n
-                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
-                n -= jit.conditional_call_elidable(p, 15, externfn, n0)
+                n -= jit.conditional_call_value(p, externfn, n0)
+                n -= jit.conditional_call_value(p, externfn, n0)
             return n
         res = self.meta_interp(f, [21, 5, 15])
         assert res == -1
@@ -161,19 +166,20 @@
         class X:
             def __init__(self, value):
                 self.value = value
-                self.triple = -1
+                self.triple = 0
+            @jit.elidable
             def _compute_triple(self):
                 self.triple = self.value * 3
                 return self.triple
             def get_triple(self):
-                return jit.conditional_call_elidable(self.triple, -1,
-                                                     X._compute_triple, self)
-        def main(n, initvalue):
+                return jit.conditional_call_value(self.triple,
+                                                  X._compute_triple, self)
+        def main(n):
             x = X(n)
-            x.triple = initvalue
             return x.get_triple() + x.get_triple()
 
-        assert self.interp_operations(main, [100, -1]) == 600
+        assert self.interp_operations(main, [100]) == 600
+        XXX
         self.check_operations_history(finish=1)   # empty history
 
 

From pypy.commits at gmail.com  Wed Nov 23 05:01:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 02:01:02 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: From
 conditional_call_value_2: found out again why we'd like the
Message-ID: <583568de.e576c20a.d6e2c.c1f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88572:1e9f2a40fd2e
Date: 2016-11-23 11:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1e9f2a40fd2e/

Log:	From conditional_call_value_2: found out again why we'd like the
	function to be only "almost" elidable

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4577,12 +4577,11 @@
         self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
 
     def test_conditional_call_value(self):
-        from rpython.rlib.jit import conditional_call_value
-        @elidable
+        from rpython.rlib.jit import conditional_call_elidable
         def g(j):
             return j + 5
         def f(i, j):
-            return conditional_call_value(i, g, j)
+            return conditional_call_elidable(i, g, j)
         res = self.interp_operations(f, [-42, 200])
         assert res == -42
         res = self.interp_operations(f, [0, 200])
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -1,5 +1,5 @@
 
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, noConst
 from rpython.rlib import jit
 
 class CallTest(object):
@@ -54,18 +54,16 @@
         self.check_resops(guard_no_exception=0)
 
     def test_cond_call_i(self):
-        @jit.elidable
         def f(n):
             return n * 200
 
         def main(n, m):
-            return jit.conditional_call_value(n, f, m)
+            return jit.conditional_call_elidable(n, f, m)
 
         assert self.interp_operations(main, [0, 10]) == 2000
         assert self.interp_operations(main, [15, 42]) == 15
 
     def test_cond_call_r(self):
-        @jit.elidable
         def f(n):
             return [n]
 
@@ -74,7 +72,7 @@
                 l = []
             else:
                 l = None
-            l = jit.conditional_call_value(l, f, n)
+            l = jit.conditional_call_elidable(l, f, n)
             return len(l)
 
         assert main(10) == 0
@@ -83,24 +81,22 @@
         assert self.interp_operations(main, [5]) == 1
 
     def test_cond_call_constant_in_pyjitpl(self):
-        @jit.elidable
         def f(a, b):
             return a + b
         def main(n):
             # this is completely constant-folded because the arguments
             # to f() are constants.
-            return jit.conditional_call_value(n, f, 40, 2)
+            return jit.conditional_call_elidable(n, f, 40, 2)
 
         assert main(12) == 12
         assert main(0) == 42
         assert self.interp_operations(main, [12]) == 12
-        self.check_operations_history(finish=1)   # empty history
+        self.check_operations_history({'finish': 1})   # empty history
         assert self.interp_operations(main, [0]) == 42
-        self.check_operations_history(finish=1)   # empty history
+        self.check_operations_history({'finish': 1})   # empty history
 
     def test_cond_call_constant_in_optimizer(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return x - 3
         class V:
@@ -110,18 +106,17 @@
             while n > 0:
                 myjitdriver.can_enter_jit(n=n, p=p, m=m)
                 myjitdriver.jit_merge_point(n=n, p=p, m=m)
-                v = V(m)
-                n -= jit.conditional_call_value(p, externfn, v.value)
+                m1 = noConst(m)
+                n -= jit.conditional_call_elidable(p, externfn, m1)
             return n
         res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
         # the COND_CALL_VALUE is constant-folded away by optimizeopt.py
-        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=0,
-                          int_sub=2)
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
 
     def test_cond_call_constant_in_optimizer_2(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return 2
         def f(n, m, p):
@@ -130,7 +125,7 @@
                 myjitdriver.jit_merge_point(n=n, p=p, m=m)
                 assert p > -1
                 assert p < 1
-                n -= jit.conditional_call_value(p, externfn, n)
+                n -= jit.conditional_call_elidable(p, externfn, n)
             return n
         res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
@@ -141,7 +136,6 @@
 
     def test_cond_call_constant_in_optimizer_3(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
-        @jit.elidable
         def externfn(x):
             return 1
         def f(n, m, p):
@@ -151,8 +145,8 @@
                 assert p > -1
                 assert p < 1
                 n0 = n
-                n -= jit.conditional_call_value(p, externfn, n0)
-                n -= jit.conditional_call_value(p, externfn, n0)
+                n -= jit.conditional_call_elidable(p, externfn, n0)
+                n -= jit.conditional_call_elidable(p, externfn, n0)
             return n
         res = self.meta_interp(f, [21, 5, 15])
         assert res == -1
@@ -167,12 +161,11 @@
             def __init__(self, value):
                 self.value = value
                 self.triple = 0
-            @jit.elidable
             def _compute_triple(self):
                 self.triple = self.value * 3
                 return self.triple
             def get_triple(self):
-                return jit.conditional_call_value(self.triple,
+                return jit.conditional_call_elidable(self.triple,
                                                   X._compute_triple, self)
         def main(n):
             x = X(n)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1218,7 +1218,7 @@
     return value    # special-cased below
 
 @specialize.call_location()
-def conditional_call_value(value, function, *args):
+def conditional_call_elidable(value, function, *args):
     """Does the same as:
 
         if value == <0 or None>:
@@ -1226,10 +1226,21 @@
         return value
 
     For the JIT.  Allows one branch which doesn't create a bridge,
-    typically used for caching.  The function must be @elidable.
-    The value and the function's return type must match and cannot
-    be a float: they must be either regular 'int', or something
-    that turns into a pointer.
+    typically used for caching.  The value and the function's return
+    type must match and cannot be a float: they must be either regular
+    'int', or something that turns into a pointer.
+
+    Even if the function is not marked @elidable, it is still treated
+    mostly like one.  The only difference is that (in heapcache.py)
+    we don't assume this function won't change anything observable.
+    This is useful for caches, as you can write:
+
+        def _compute_and_cache(...):
+            self.cache = ...compute...
+            return self.cache
+
+        x = jit.conditional_call_elidable(self.cache, _compute_and_cache, ...)
+
     """
     if we_are_jitted():
         return _jit_conditional_call_value(value, function, *args)
@@ -1243,7 +1254,7 @@
                 value = function(*args)
                 assert not isinstance(value, int)
         return value
-conditional_call_value._always_inline_ = True
+conditional_call_elidable._always_inline_ = True
 
 class ConditionalCallEntry(ExtRegistryEntry):
     _about_ = _jit_conditional_call, _jit_conditional_call_value
@@ -1253,10 +1264,6 @@
                                                  args_s[1], args_s[2:])
         if self.instance == _jit_conditional_call_value:
             from rpython.annotator import model as annmodel
-            func = args_s[1].const
-            assert getattr(func, '_elidable_function_', None), (
-                "%r used in jit.conditional_call_value() should be "
-                "@jit.elidable" % (func,))
             return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -3,8 +3,8 @@
 from rpython.conftest import option
 from rpython.annotator.model import UnionError
 from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote,
-    JitHintError, oopspec, isconstant, conditional_call, conditional_call_value,
-    elidable, unroll_safe, dont_look_inside,
+    JitHintError, oopspec, isconstant, conditional_call,
+    elidable, unroll_safe, dont_look_inside, conditional_call_elidable,
     enter_portal_frame, leave_portal_frame)
 from rpython.rlib.rarithmetic import r_uint
 from rpython.rtyper.test.tool import BaseRtypingTest
@@ -302,12 +302,11 @@
         mix.getgraph(later, [annmodel.s_Bool], annmodel.s_None)
         mix.finish()
 
-    def test_conditional_call_value(self):
-        @elidable
+    def test_conditional_call_elidable(self):
         def g(x, y):
             return x - y + 5
         def f(n, x, y):
-            return conditional_call_value(n, g, x, y)
+            return conditional_call_elidable(n, g, x, y)
         assert f(0, 1000, 100) == 905
         res = self.interpret(f, [0, 1000, 100])
         assert res == 905

From pypy.commits at gmail.com  Wed Nov 23 05:05:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 02:05:11 -0800 (PST)
Subject: [pypy-commit] pypy default: make INT_EQ and INT_NE more similar
Message-ID: <583569d7.2a0bc30a.71fd5.577d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88573:55a9404c80d6
Date: 2016-11-23 11:04 +0100
http://bitbucket.org/pypy/pypy/changeset/55a9404c80d6/

Log:	make INT_EQ and INT_NE more similar

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -399,9 +399,9 @@
 
     def optimize_INT_EQ(self, op):
         arg0 = self.get_box_replacement(op.getarg(0))
+        b1 = self.getintbound(arg0)
         arg1 = self.get_box_replacement(op.getarg(1))
-        b1 = self.getintbound(op.getarg(0))
-        b2 = self.getintbound(op.getarg(1))
+        b2 = self.getintbound(arg1)
         if b1.known_gt(b2):
             self.make_constant_int(op, 0)
         elif b1.known_lt(b2):

From pypy.commits at gmail.com  Wed Nov 23 05:26:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 02:26:29 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Pass tests
Message-ID: <58356ed5.c4251c0a.ab6d3.9cee@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88574:507376cf760c
Date: 2016-11-23 11:25 +0100
http://bitbucket.org/pypy/pypy/changeset/507376cf760c/

Log:	Pass tests

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -596,6 +596,18 @@
             op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         return self.emit(op)
 
+    def optimize_COND_CALL_VALUE_I(self, op):
+        info = self.getnullness(op.getarg(0))
+        if info == INFO_NONNULL:
+            self.make_equal_to(op, op.getarg(0))
+            self.last_emitted_operation = REMOVED
+            return
+        if info == INFO_NULL:
+            opnum = OpHelpers.call_for_type(op.type)
+            op = self.replace_op_with(op, opnum, args=op.getarglist()[1:])
+        return self.emit(op)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
     def _optimize_nullness(self, op, box, expect_nonnull):
         info = self.getnullness(box)
         if info == INFO_NONNULL:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8674,6 +8674,59 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_with_a_constant_i(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(0, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i2 = call_i(123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_with_a_constant_i2(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(12, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        escape_n(12)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r1(self):
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        jump(p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_cond_call_r2(self):
+        ops = """
+        [p1]
+        guard_nonnull(p1) []
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        guard_nonnull(p1) []
+        p3 = escape_r(p1)
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -438,6 +438,10 @@
                     oopspecindex=EffectInfo.OS_INT_PY_MOD)
     int_py_mod_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
 
+    FUNC = lltype.FuncType([], llmemory.GCREF)
+    ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE)
+    plain_r_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
+
     namespace = locals()
 
 

From pypy.commits at gmail.com  Wed Nov 23 05:46:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 02:46:31 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: progress
Message-ID: <58357387.542e1c0a.d6cc.a327@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88575:330341af61ab
Date: 2016-11-23 11:45 +0100
http://bitbucket.org/pypy/pypy/changeset/330341af61ab/

Log:	progress

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -261,9 +261,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
-    def _can_optimize_call_pure(self, op):
+    def _can_optimize_call_pure(self, op, start_index=0):
         arg_consts = []
-        for i in range(op.numargs()):
+        for i in range(start_index, op.numargs()):
             arg = op.getarg(i)
             const = self.optimizer.get_constant_box(arg)
             if const is None:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -597,6 +597,14 @@
         return self.emit(op)
 
     def optimize_COND_CALL_VALUE_I(self, op):
+        # this removes a COND_CALL_VALUE with all constant arguments
+        # (ignoring the 'value' in arg0)
+        result = self._can_optimize_call_pure(op, start_index=1)
+        if result is not None:
+            self.make_constant(op, result)
+            self.last_emitted_operation = REMOVED
+            return
+        # otherwise, look if we know the nullness of the first argument
         info = self.getnullness(op.getarg(0))
         if info == INFO_NONNULL:
             self.make_equal_to(op, op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, TreeLoop
+from rpython.jit.metainterp.history import ConstPtr
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import build_opt_chain
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
@@ -8727,6 +8728,22 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_r3(self):
+        arg_consts = [ConstInt(i) for i in (123, 4, 5, 6)]
+        call_pure_results = {tuple(arg_consts): ConstPtr(self.myptr)}
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, 4, 5, 6, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        p3 = escape_r(ConstPtr(myptr))
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -3098,7 +3098,7 @@
             return resbox_as_const
         # not all constants (so far): turn CALL into CALL_PURE, which might
         # be either removed later by optimizeopt or turned back into CALL.
-        arg_consts = [executor.constant_from_op(a) for a in argboxes]
+        arg_consts = [executor.constant_from_op(a) for a in normargboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
         if is_cond_value:
             return op       # but COND_CALL_VALUE remains

From pypy.commits at gmail.com  Wed Nov 23 05:57:13 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Nov 2016 02:57:13 -0800 (PST)
Subject: [pypy-commit] pypy default: fix sandbox in the case where an os
 function is called directly via rposix, as
Message-ID: <58357609.8a29c20a.df6a.dd45@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88576:f8b82bd65934
Date: 2016-11-23 11:55 +0100
http://bitbucket.org/pypy/pypy/changeset/f8b82bd65934/

Log:	fix sandbox in the case where an os function is called directly via
	rposix, as opposed to via os.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -323,6 +323,8 @@
 def register_replacement_for(replaced_function, sandboxed_name=None):
     def wrap(func):
         from rpython.rtyper.extregistry import ExtRegistryEntry
+        # to support calling func directly
+        func._sandbox_external_name = sandboxed_name
         class ExtRegistry(ExtRegistryEntry):
             _about_ = replaced_function
             def compute_annotation(self):
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -65,6 +65,24 @@
     f.close()
     assert tail == ""
 
+def test_open_dup_rposix():
+    from rpython.rlib import rposix
+    def entry_point(argv):
+        fd = rposix.open("/tmp/foobar", os.O_RDONLY, 0777)
+        assert fd == 77
+        fd2 = rposix.dup(fd)
+        assert fd2 == 78
+        return 0
+
+    exe = compile(entry_point)
+    g, f = run_in_subprocess(exe)
+    expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
+    g.close()
+    tail = f.read()
+    f.close()
+    assert tail == ""
+
 def test_read_write():
     def entry_point(argv):
         fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)

From pypy.commits at gmail.com  Wed Nov 23 05:58:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 02:58:51 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: next test passing
Message-ID: <5835766b.c8111c0a.5fdd4.a7f2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88577:dbe7fb152fc7
Date: 2016-11-23 11:58 +0100
http://bitbucket.org/pypy/pypy/changeset/dbe7fb152fc7/

Log:	next test passing

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -151,13 +151,13 @@
             self._pure_operations[opnum] = recentops = RecentPureOps()
         return recentops
 
-    def optimize_CALL_PURE_I(self, op):
+    def optimize_call_pure(self, op, start_index=0):
         # Step 1: check if all arguments are constant
-        for arg in op.getarglist():
-            self.optimizer.force_box(arg)
+        for i in range(start_index, op.numargs()):
+            self.optimizer.force_box(op.getarg(i))
             # XXX hack to ensure that virtuals that are
             #     constant are presented that way
-        result = self._can_optimize_call_pure(op)
+        result = self._can_optimize_call_pure(op, start_index=start_index)
         if result is not None:
             # this removes a CALL_PURE with all constant arguments.
             self.make_constant(op, result)
@@ -168,26 +168,35 @@
         # CALL_PURE.
         for pos in self.call_pure_positions:
             old_op = self.optimizer._newoperations[pos]
-            if self.optimize_call_pure(op, old_op):
+            if self.optimize_call_pure_old(op, old_op):
                 return
         if self.extra_call_pure:
             for i, old_op in enumerate(self.extra_call_pure):
-                if self.optimize_call_pure(op, old_op):
+                if self.optimize_call_pure_old(op, old_op):
                     if isinstance(old_op, PreambleOp):
                         old_op = self.optimizer.force_op_from_preamble(old_op)
                         self.extra_call_pure[i] = old_op
                     return
 
-        # replace CALL_PURE with just CALL
-        opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = self.optimizer.replace_op_with(op, opnum)
+        # replace CALL_PURE with just CALL (but keep COND_CALL_VALUE)
+        if start_index == 0:
+            opnum = OpHelpers.call_for_descr(op.getdescr())
+            newop = self.optimizer.replace_op_with(op, opnum)
+        else:
+            newop = op
         return self.emit_result(CallPureOptimizationResult(self, newop))
 
+    def optimize_CALL_PURE_I(self, op):
+        return self.optimize_call_pure(op)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
-    def optimize_call_pure(self, op, old_op):
+    def optimize_COND_CALL_VALUE_I(self, op):
+        return self.optimize_call_pure(op, start_index=1)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
+    def optimize_call_pure_old(self, op, old_op):
         if (op.numargs() != old_op.numargs() or
             op.getdescr() is not old_op.getdescr()):
             return False
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -611,7 +611,7 @@
             self.last_emitted_operation = REMOVED
             return
         if info == INFO_NULL:
-            opnum = OpHelpers.call_for_type(op.type)
+            opnum = OpHelpers.call_pure_for_type(op.type)
             op = self.replace_op_with(op, opnum, args=op.getarglist()[1:])
         return self.emit(op)
     optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1335,6 +1335,16 @@
         return rop.CALL_N
 
     @staticmethod
+    def call_pure_for_type(tp):
+        if tp == 'i':
+            return rop.CALL_PURE_I
+        elif tp == 'r':
+            return rop.CALL_PURE_R
+        elif tp == 'f':
+            return rop.CALL_PURE_F
+        return rop.CALL_PURE_N
+
+    @staticmethod
     def is_guard(opnum):
         return rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST
 
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -131,7 +131,7 @@
         assert res == -1
         # optimizer: the COND_CALL_VALUE is turned into a regular
         # CALL_PURE, which itself becomes a CALL
-        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+        self.check_resops(call_pure_i=0, cond_call_value_i=0, call_i=2,
                           int_sub=2)
 
     def test_cond_call_constant_in_optimizer_3(self):

From pypy.commits at gmail.com  Wed Nov 23 06:15:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 03:15:49 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: tweaks
Message-ID: <58357a65.90121c0a.32469.ae87@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88578:71709aa3ea99
Date: 2016-11-23 12:15 +0100
http://bitbucket.org/pypy/pypy/changeset/71709aa3ea99/

Log:	tweaks

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -168,11 +168,11 @@
         # CALL_PURE.
         for pos in self.call_pure_positions:
             old_op = self.optimizer._newoperations[pos]
-            if self.optimize_call_pure_old(op, old_op):
+            if self.optimize_call_pure_old(op, old_op, start_index):
                 return
         if self.extra_call_pure:
             for i, old_op in enumerate(self.extra_call_pure):
-                if self.optimize_call_pure_old(op, old_op):
+                if self.optimize_call_pure_old(op, old_op, start_index):
                     if isinstance(old_op, PreambleOp):
                         old_op = self.optimizer.force_op_from_preamble(old_op)
                         self.extra_call_pure[i] = old_op
@@ -196,11 +196,12 @@
         return self.optimize_call_pure(op, start_index=1)
     optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
 
-    def optimize_call_pure_old(self, op, old_op):
+    def optimize_call_pure_old(self, op, old_op, start_index):
         if (op.numargs() != old_op.numargs() or
             op.getdescr() is not old_op.getdescr()):
             return False
-        for i, box in enumerate(old_op.getarglist()):
+        for i in range(start_index, old_op.numargs()):
+            box = old_op.getarg(i)
             if not self.get_box_replacement(op.getarg(i)).same_box(box):
                 break
         else:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -597,14 +597,7 @@
         return self.emit(op)
 
     def optimize_COND_CALL_VALUE_I(self, op):
-        # this removes a COND_CALL_VALUE with all constant arguments
-        # (ignoring the 'value' in arg0)
-        result = self._can_optimize_call_pure(op, start_index=1)
-        if result is not None:
-            self.make_constant(op, result)
-            self.last_emitted_operation = REMOVED
-            return
-        # otherwise, look if we know the nullness of the first argument
+        # look if we know the nullness of the first argument
         info = self.getnullness(op.getarg(0))
         if info == INFO_NONNULL:
             self.make_equal_to(op, op.getarg(0))

From pypy.commits at gmail.com  Wed Nov 23 09:42:15 2016
From: pypy.commits at gmail.com (mattip)
Date: Wed, 23 Nov 2016 06:42:15 -0800 (PST)
Subject: [pypy-commit] pypy default: test, fix user-created heaptype,
 issue #2434
Message-ID: <5835aac7.0d1a1c0a.c1daa.0260@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88579:98f8c7e783db
Date: 2016-11-23 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/98f8c7e783db/

Log:	test, fix user-created heaptype, issue #2434

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -331,12 +331,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -469,7 +469,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -852,14 +852,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS

From pypy.commits at gmail.com  Wed Nov 23 09:55:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 06:55:15 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: More tests, more fixes
Message-ID: <5835add3.0a74c20a.6f7f9.40c7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88580:be014558712e
Date: 2016-11-23 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/be014558712e/

Log:	More tests, more fixes

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -197,13 +197,17 @@
     optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
 
     def optimize_call_pure_old(self, op, old_op, start_index):
-        if (op.numargs() != old_op.numargs() or
-            op.getdescr() is not old_op.getdescr()):
+        if op.getdescr() is not old_op.getdescr():
             return False
-        for i in range(start_index, old_op.numargs()):
+        # this will match a call_pure and a cond_call_value with
+        # the same function and arguments
+        j = start_index
+        old_start_index = OpHelpers.is_cond_call_value(old_op.opnum)
+        for i in range(old_start_index, old_op.numargs()):
             box = old_op.getarg(i)
-            if not self.get_box_replacement(op.getarg(i)).same_box(box):
+            if not self.get_box_replacement(op.getarg(j)).same_box(box):
                 break
+            j += 1
         else:
             # all identical
             # this removes a CALL_PURE that has the same (non-constant)
@@ -260,10 +264,17 @@
             # don't move call_pure_with_exception in the short preamble...
             # issue #2015
 
+            # Also, don't move cond_call_value in the short preamble.
+            # The issue there is that it's usually pointless to try to
+            # because the 'value' argument is typically not a loop
+            # invariant, and would really need to be in order to end up
+            # in the short preamble.  Maybe the code works anyway in the
+            # other rare case, but better safe than sorry and don't try.
             effectinfo = op.getdescr().get_extra_info()
             if not effectinfo.check_can_raise(ignore_memoryerror=True):
                 assert rop.is_call(op.opnum)
-                sb.add_pure_op(op)
+                if not OpHelpers.is_cond_call_value(op.opnum):
+                    sb.add_pure_op(op)
 
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -115,6 +115,29 @@
         self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
                            'jump': 1})
 
+    def test_cond_call_constant_in_optimizer_1(self):
+        # same as test_cond_call_constant_in_optimizer, but the 'value'
+        # argument changes
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return x - 3
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                m1 = noConst(m)
+                n -= jit.conditional_call_elidable(p, externfn, m1)
+            return n
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        # the COND_CALL_VALUE is constant-folded away by optimizeopt.py
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
+
     def test_cond_call_constant_in_optimizer_2(self):
         myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
         def externfn(x):
@@ -148,12 +171,12 @@
                 n -= jit.conditional_call_elidable(p, externfn, n0)
                 n -= jit.conditional_call_elidable(p, externfn, n0)
             return n
-        res = self.meta_interp(f, [21, 5, 15])
+        res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
         # same as test_cond_call_constant_in_optimizer_2, but the two
         # intermediate CALL_PUREs are replaced with only one, because
         # they are called with the same arguments
-        self.check_resops(call_pure_i=0, cond_call_pure_i=0, call_i=2,
+        self.check_resops(call_pure_i=0, cond_call_value_i=0, call_i=2,
                           int_sub=4)
 
     def test_cond_call_constant_in_optimizer_4(self):
@@ -167,13 +190,71 @@
             def get_triple(self):
                 return jit.conditional_call_elidable(self.triple,
                                                   X._compute_triple, self)
+
+        myjitdriver = jit.JitDriver(greens = [], reds = 'auto')
         def main(n):
-            x = X(n)
-            return x.get_triple() + x.get_triple()
+            total = 0
+            while n > 1:
+                myjitdriver.jit_merge_point()
+                x = X(n)
+                total += x.get_triple() + x.get_triple() + x.get_triple()
+                n -= 10
+            return total
 
-        assert self.interp_operations(main, [100]) == 600
-        XXX
-        self.check_operations_history(finish=1)   # empty history
+        res = self.meta_interp(main, [100])
+        assert res == main(100)
+        # remaining: only the first call to get_triple(), as a call_i
+        # because we know that x.triple == 0 here.  The remaining calls
+        # are removed because equal to the first one.
+        self.check_resops(call_i=2, cond_call_value_i=0)
+
+    def test_cond_call_multiple_in_optimizer_1(self):
+        # test called several times with the same arguments, but
+        # the condition is not available to the short preamble.
+        # This means that the second cond_call_value after unrolling
+        # can't be removed.
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2000      # never actually called
+        @jit.dont_look_inside
+        def randomish(p):
+            return p + 1
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                n -= jit.conditional_call_elidable(randomish(p), externfn, m)
+            return n
+        assert f(21, 5, 1) == -1
+        res = self.meta_interp(f, [21, 5, 1])
+        assert res == -1
+        self.check_resops(call_pure_i=0, cond_call_value_i=2,
+                          call_i=2,    # randomish()
+                          int_sub=2)
+
+    def test_cond_call_multiple_in_optimizer_2(self):
+        # test called several times with the same arguments.  Ideally
+        # we would like them to be consolidated into one call even if
+        # the 'value' are different but available from the short
+        # preamble.  We don't do it so far---it's a mess, because the
+        # short preamble is supposed to depend only on loop-invariant
+        # things, and 'value' is (most of the time) not loop-invariant.
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2      # called only the first time
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                p = jit.conditional_call_elidable(p, externfn, m)
+                n -= p
+            return n
+        assert f(219, 5, 0) == -1
+        res = self.meta_interp(f, [219, 5, 0])
+        assert res == -1
+        self.check_resops(call_pure_i=0,
+                          cond_call_value_i=2,   # ideally 1, but see above
+                          int_sub=2)
 
 
 class TestCall(LLJitMixin, CallTest):

From pypy.commits at gmail.com  Wed Nov 23 10:36:23 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Nov 2016 07:36:23 -0800 (PST)
Subject: [pypy-commit] pypy default: issue 2435 testing: make sure to ignore
 the overridden __getitem__ method on
Message-ID: <5835b777.849c1c0a.38a6c.1477@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88581:97648f46bf4e
Date: 2016-11-23 16:18 +0100
http://bitbucket.org/pypy/pypy/changeset/97648f46bf4e/

Log:	issue 2435 testing: make sure to ignore the overridden __getitem__
	method on subclasses pass as **kwargs. This is bug-to-bug
	compatibility with CPython.

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -111,7 +111,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -125,7 +127,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -355,7 +359,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -374,7 +378,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,6 +120,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type_where(self, cls, name):
+        return 'hopefully not needed', getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -805,3 +811,19 @@
             assert str(e) == "myerror"
         else:
             assert False, "Expected TypeError"
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -61,16 +61,24 @@
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_str,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
+                                                  '__getitem__')
+    return w_getitem
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
+                                                  '__getitem__')
+    return w_getitem
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
+                                                  '__getitem__')
+    return w_getitem
+
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:

From pypy.commits at gmail.com  Wed Nov 23 11:26:05 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 23 Nov 2016 08:26:05 -0800 (PST)
Subject: [pypy-commit] pypy default: test and fix: PyDict_Update is not
 actually the same as dict.update in CPython
Message-ID: <5835c31d.c4811c0a.2dd69.2fa5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88582:436f95cd8184
Date: 2016-11-23 17:25 +0100
http://bitbucket.org/pypy/pypy/changeset/436f95cd8184/

Log:	test and fix: PyDict_Update is not actually the same as dict.update
	in CPython (that would be too simple)

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -199,3 +210,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+

From pypy.commits at gmail.com  Wed Nov 23 11:47:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 08:47:31 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Remove
 'supports_cond_call_value', because now it is expected from all backends
Message-ID: <5835c823.ca10c20a.5aae6.6da7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88583:184d01694ab1
Date: 2016-11-23 16:43 +0100
http://bitbucket.org/pypy/pypy/changeset/184d01694ab1/

Log:	Remove 'supports_cond_call_value', because now it is expected from
	all backends

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -326,7 +326,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -596,8 +596,6 @@
                                       size, 1, ofs)
 
     def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
         descr = op.getdescr()
         if descr is None:
             return
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -216,7 +216,6 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
-    supports_cond_call_value = True
 
     translate_support_code = None
 
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,7 +16,6 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
-    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,9 +2448,6 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
-        if not self.cpu.supports_cond_call_value:
-            py.test.skip("missing supports_cond_call_value")
-
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -711,11 +711,6 @@
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
 
-    def filter(self, builder):
-        if not builder.cpu.supports_cond_call_value and \
-           self.opnum == rop.COND_CALL_VALUE_I:
-            raise test_random.CannotProduceOperation
-
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -20,11 +20,6 @@
     r = Random()
     r.jumpahead(piece*99999999)
     OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,7 +16,6 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
-    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False

From pypy.commits at gmail.com  Wed Nov 23 11:47:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 08:47:33 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: add 'cond_call_value'
 in the ARM backend
Message-ID: <5835c825.ca10c20a.5aae6.6dad@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88584:208a94fe7827
Date: 2016-11-23 17:46 +0100
http://bitbucket.org/pypy/pypy/changeset/208a94fe7827/

Log:	add 'cond_call_value' in the ARM backend

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg

From pypy.commits at gmail.com  Wed Nov 23 12:33:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 09:33:21 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: PPC support for
 cond_call_value
Message-ID: <5835d2e1.581d1c0a.9dd87.4fa8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88585:09b115d2a587
Date: 2016-11-23 18:30 +0100
http://bitbucket.org/pypy/pypy/changeset/09b115d2a587/

Log:	PPC support for cond_call_value

diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -656,10 +656,12 @@
     _COND_CALL_SAVE_REGS = [r.r3, r.r4, r.r5, r.r6, r.r12]
 
     def emit_cond_call(self, op, arglocs, regalloc):
+        resloc = arglocs[0]
+        arglocs = arglocs[1:]
+
         fcond = self.guard_success_cc
         self.guard_success_cc = c.cond_none
         assert fcond != c.cond_none
-        fcond = c.negate(fcond)
 
         jmp_adr = self.mc.get_relative_pos()
         self.mc.trap()        # patched later to a 'bc'
@@ -691,7 +693,10 @@
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.bl_abs(cond_call_adr)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  If we have a result, move
+        # it from r2 to its expected location.
+        if resloc is not None:
+            self.mc.mr(resloc.value, r.SCRATCH2.value)
         relative_target = self.mc.currpos() - jmp_adr
         pmc = OverwritingBuilder(self.mc, jmp_adr, 1)
         BI, BO = c.encoding[fcond]
@@ -701,6 +706,9 @@
         # guard_no_exception too
         self.previous_cond_call_jcond = jmp_adr, BI, BO
 
+    emit_cond_call_value_i = emit_cond_call
+    emit_cond_call_value_r = emit_cond_call
+
 
 class FieldOpAssembler(object):
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -315,6 +315,7 @@
         #   * r2 is the gcmap
         #   * the old value of these regs must already be stored in the jitframe
         #   * on exit, all registers are restored from the jitframe
+        #   * the result of the call, if any, is moved to r2
 
         mc = PPCBuilder()
         self.mc = mc
@@ -347,7 +348,11 @@
         # Finish
         self._reload_frame_if_necessary(mc)
 
+        # Move the result, if any, to r2
+        mc.mr(r.SCRATCH2.value, r.r3.value)
+
         mc.mtlr(r.RCS1.value)     # restore LR
+
         self._pop_core_regs_from_jitframe(mc, saved_regs)
         if supports_floats:
             self._pop_fp_regs_from_jitframe(mc)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1038,14 +1038,34 @@
 
     def prepare_cond_call(self, op):
         self.load_condition_into_cc(op.getarg(0))
-        locs = []
+        self.assembler.guard_success_cc = c.negate(
+            self.assembler.guard_success_cc)
+        # ^^^ if arg0==0, we jump over the next block of code (the call)
+        locs = [None]
         # support between 0 and 4 integer arguments
         assert 2 <= op.numargs() <= 2 + 4
         for i in range(1, op.numargs()):
             loc = self.loc(op.getarg(i))
             assert loc.type != FLOAT
             locs.append(loc)
-        return locs
+        return locs     # [None, function, args...]
+
+    def prepare_cond_call_value_i(self, op):
+        x = self.ensure_reg(op.getarg(0))
+        self.load_condition_into_cc(op.getarg(0))
+        self.rm.force_allocate_reg(op, selected_reg=x)   # spilled if survives
+        # ^^^ if arg0!=0, we jump over the next block of code (the call)
+        locs = [x]
+        # support between 0 and 4 integer arguments
+        assert 2 <= op.numargs() <= 2 + 4
+        for i in range(1, op.numargs()):
+            loc = self.loc(op.getarg(i))
+            assert loc.type != FLOAT
+            locs.append(loc)
+        return locs     # [res, function, args...]
+
+    prepare_cond_call_value_r = prepare_cond_call_value_i
+
 
 def notimplemented(self, op):
     msg = '[PPC/regalloc] %s not implemented\n' % op.getopname()

From pypy.commits at gmail.com  Wed Nov 23 12:39:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 09:39:02 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Use
 conditional_call_elidable instead of call_shortcut here
Message-ID: <5835d436.572d1c0a.75dca.536c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88586:f4289ee8dfa3
Date: 2016-11-23 18:38 +0100
http://bitbucket.org/pypy/pypy/changeset/f4289ee8dfa3/

Log:	Use conditional_call_elidable instead of call_shortcut here

diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py
--- a/rpython/rtyper/lltypesystem/rstr.py
+++ b/rpython/rtyper/lltypesystem/rstr.py
@@ -371,23 +371,21 @@
     @staticmethod
     def ll_strhash(s):
         if s:
-            return LLHelpers._ll_strhash(s)
+            return jit.conditional_call_elidable(s.hash,
+                                                 LLHelpers._ll_strhash, s)
         else:
             return 0
 
     @staticmethod
-    @jit.elidable
-    @jit.call_shortcut
     def _ll_strhash(s):
         # unlike CPython, there is no reason to avoid to return -1
         # but our malloc initializes the memory to zero, so we use zero as the
-        # special non-computed-yet value.
-        x = s.hash
+        # special non-computed-yet value.  Also, jit.conditional_call_elidable
+        # always checks for zero, for now.
+        x = _hash_string(s.chars)
         if x == 0:
-            x = _hash_string(s.chars)
-            if x == 0:
-                x = 29872897
-            s.hash = x
+            x = 29872897
+        s.hash = x
         return x
 
     @staticmethod

From pypy.commits at gmail.com  Wed Nov 23 12:42:55 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 09:42:55 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: skip micronumpy tests
Message-ID: <5835d51f.c4251c0a.ab6d3.53ca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88587:5a606b09d9e7
Date: 2016-11-23 17:42 +0000
http://bitbucket.org/pypy/pypy/changeset/5a606b09d9e7/

Log:	skip micronumpy tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -16,6 +16,7 @@
         return not detect_simd_z()
     return True
 
+ at py.test.mark.skipif(True, reason='no _numpypy on pypy3')
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),

From pypy.commits at gmail.com  Wed Nov 23 12:45:04 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 09:45:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: 2to3fy test
Message-ID: <5835d5a0.46831c0a.c8125.5489@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88588:bf7b4bdeda0e
Date: 2016-11-23 17:44 +0000
http://bitbucket.org/pypy/pypy/changeset/bf7b4bdeda0e/

Log:	2to3fy test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py
--- a/pypy/module/pypyjit/test_pypy_c/test_shift.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py
@@ -47,9 +47,9 @@
             res = 0
             a = 0
             while a < 300:
-                res1 = a/b     # ID: div
-                res2 = a/2     # ID: shift
-                res3 = a/11    # ID: mul
+                res1 = a // b     # ID: div
+                res2 = a // 2     # ID: shift
+                res3 = a // 11    # ID: mul
                 res += res1 + res2 + res3
                 a += 1
             return res

From pypy.commits at gmail.com  Wed Nov 23 14:03:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 11:03:45 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix import
Message-ID: <5835e811.08301c0a.670e7.7725@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88589:ff4b1d16a369
Date: 2016-11-23 19:03 +0000
http://bitbucket.org/pypy/pypy/changeset/ff4b1d16a369/

Log:	fix import

diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -106,7 +106,7 @@
 
 class AppTestFetch(AppTestCpythonExtensionBase):
     def setup_class(cls):
-        from pypy.module.imp.test.support import get_special_char
+        from pypy.interpreter.test.test_fsencode import get_special_char
         space = cls.space
         cls.special_char = get_special_char()
         cls.w_special_char = space.wrap(cls.special_char)
@@ -380,7 +380,7 @@
         "XXX seems to pass, but doesn't: 'py.test -s' shows errors in PyObject_Free")
     def test_GetSetExcInfo(self):
         import sys
-        if self.runappdirect and (sys.version_info.major < 3 or 
+        if self.runappdirect and (sys.version_info.major < 3 or
                                   sys.version_info.minor < 3):
             skip('PyErr_{GS}etExcInfo introduced in python 3.3')
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Wed Nov 23 14:29:58 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 11:29:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Fix tests for CPython -A
Message-ID: <5835ee36.45f6c20a.f25f.b408@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88590:b5e49d334122
Date: 2016-11-23 19:29 +0000
http://bitbucket.org/pypy/pypy/changeset/b5e49d334122/

Log:	Fix tests for CPython -A

diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -4,15 +4,6 @@
 class AppTestArrayModule(AppTestCpythonExtensionBase):
     enable_leak_checking = True
 
-    def setup_class(cls):
-        from rpython.tool.udir import udir
-        AppTestCpythonExtensionBase.setup_class.im_func(cls)
-        if option.runappdirect:
-            cls.w_udir = str(udir)
-        else:
-            cls.w_udir = cls.space.wrap(str(udir))
-
-
     def test_basic(self):
         module = self.import_module(name='array')
         arr = module.array('i', [1,2,3])
@@ -105,30 +96,28 @@
         # Not really part of array, refactor
         import struct
         module = self.import_module(name='array')
-        val = module.readbuffer_as_string('abcd')
-        assert val == 'abcd'
-        val = module.readbuffer_as_string(u'\u03a3')
-        assert val is not None
+        val = module.readbuffer_as_string(b'abcd')
+        assert val == b'abcd'
 
     def test_readinto(self):
         module = self.import_module(name='array')
-        a = module.array('c')
-        a.fromstring('0123456789')
+        a = module.array('B')
+        a.fromstring(b'0123456789')
         filename = self.udir + "/_test_file"
         f = open(filename, 'w+b')
-        f.write('foobar')
+        f.write(b'foobar')
         f.seek(0)
         n = f.readinto(a)
         f.close()
         assert n == 6
         assert len(a) == 10
-        assert a.tostring() == 'foobar6789'
+        assert a.tostring() == b'foobar6789'
 
     def test_iowrite(self):
         module = self.import_module(name='array')
         from io import BytesIO
-        a = module.array('c')
-        a.fromstring('0123456789')
+        a = module.array('B')
+        a.fromstring(b'0123456789')
         fd = BytesIO()
         # only test that it works
         fd.write(a)

From pypy.commits at gmail.com  Wed Nov 23 14:50:53 2016
From: pypy.commits at gmail.com (wlav)
Date: Wed, 23 Nov 2016 11:50:53 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: simplified dispatch for improved
 performance
Message-ID: <5835f31d.4c9d1c0a.45b6.8970@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88591:d4eede32eec0
Date: 2016-11-23 11:42 -0800
http://bitbucket.org/pypy/pypy/changeset/d4eede32eec0/

Log:	simplified dispatch for improved performance

diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -25,11 +25,32 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, h = 0, l = -1, s = '', vp = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+        self.tc      = tc
         self._handle = h
         self._long   = l
         self._string = s
-        self._voidp  = vp
+        self._voidp  = p
+
+class _ArgH(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'h', h = val)
+
+class _ArgL(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'l', l = val)
+
+class _ArgS(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 's', s = val)
+
+class _ArgP(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'p', p = val)
 
 # For the loadable CAPI, the calls start and end in RPython. Therefore, the standard
 # _call of W_CTypeFunc, which expects wrapped objects, does not quite work: some
@@ -59,14 +80,18 @@
                 argtype = self.fargs[i]
                 # the following is clumsy, but the data types used as arguments are
                 # very limited, so it'll do for now
-                if isinstance(argtype, ctypeprim.W_CTypePrimitiveSigned):
+                if obj.tc == 'l':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveSigned)
                     misc.write_raw_signed_data(data, rffi.cast(rffi.LONG, obj._long), argtype.size)
-                elif isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned):
+                elif obj.tc == 'h':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveUnsigned)
                     misc.write_raw_unsigned_data(data, rffi.cast(rffi.ULONG, obj._handle), argtype.size)
-                elif obj._voidp != rffi.cast(rffi.VOIDP, 0):
+                elif obj.tc == 'p':
+                    assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
                 else:    # only other use is sring
+                    assert obj.tc == 's'
                     n = len(obj._string)
                     assert raw_string == rffi.cast(rffi.CCHARP, 0)
                     # XXX could use rffi.get_nonmovingbuffer_final_null()
@@ -298,87 +323,87 @@
 
 # name to opaque C++ scope representation ------------------------------------
 def c_num_scopes(space, cppscope):
-    return space.int_w(call_capi(space, 'num_scopes', [_Arg(h=cppscope.handle)]))
+    return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)]))
 def c_scope_name(space, cppscope, iscope):
-    args = [_Arg(h=cppscope.handle), _Arg(l=iscope)]
+    args = [_ArgH(cppscope.handle), _ArgL(iscope)]
     return charp2str_free(space, call_capi(space, 'scope_name', args))
 
 def c_resolve_name(space, name):
-    return charp2str_free(space, call_capi(space, 'resolve_name', [_Arg(s=name)]))
+    return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)]))
 def c_get_scope_opaque(space, name):
-    return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_Arg(s=name)])))
+    return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)])))
 def c_actual_class(space, cppclass, cppobj):
-    args = [_Arg(h=cppclass.handle), _Arg(h=cppobj)]
+    args = [_ArgH(cppclass.handle), _ArgH(cppobj)]
     return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args)))
 
 # memory management ----------------------------------------------------------
 def c_allocate(space, cppclass):
-    return _cdata_to_cobject(space, call_capi(space, 'allocate', [_Arg(h=cppclass.handle)]))
+    return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)]))
 def c_deallocate(space, cppclass, cppobject):
-    call_capi(space, 'deallocate', [_Arg(h=cppclass.handle), _Arg(h=cppobject)])
+    call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)])
 def c_destruct(space, cppclass, cppobject):
-    call_capi(space, 'destruct', [_Arg(h=cppclass.handle), _Arg(h=cppobject)])
+    call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)])
 
 # method/function dispatching ------------------------------------------------
 def c_call_v(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     call_capi(space, 'call_v', args)
 def c_call_b(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.UCHAR, space.c_uint_w(call_capi(space, 'call_b', args)))
 def c_call_c(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.CHAR, space.str_w(call_capi(space, 'call_c', args))[0])
 def c_call_h(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.SHORT, space.int_w(call_capi(space, 'call_h', args)))
 def c_call_i(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.INT, space.c_int_w(call_capi(space, 'call_i', args)))
 def c_call_l(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.LONG, space.int_w(call_capi(space, 'call_l', args)))
 def c_call_ll(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.LONGLONG, space.r_longlong_w(call_capi(space, 'call_ll', args)))
 def c_call_f(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.FLOAT, r_singlefloat(space.float_w(call_capi(space, 'call_f', args))))
 def c_call_d(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return _cdata_to_ptr(space, call_capi(space, 'call_r', args))
 def c_call_s(space, cppmethod, cppobject, nargs, cargs):
     length = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
         w_cstr = call_capi(space, 'call_s',
-            [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs),
-             _Arg(vp=rffi.cast(rffi.VOIDP, length))])
+            [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs),
+             _ArgP(rffi.cast(rffi.VOIDP, length))])
         cstr_len = intmask(length[0])
     finally:
         lltype.free(length, flavor='raw')
     return _cdata_to_ccharp(space, w_cstr), cstr_len
 
 def c_constructor(space, cppmethod, cppobject, nargs, cargs):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
     return _cdata_to_cobject(space, call_capi(space, 'constructor', args))
 def c_call_o(space, cppmethod, cppobject, nargs, cargs, cppclass):
-    args = [_Arg(h=cppmethod), _Arg(h=cppobject), _Arg(l=nargs), _Arg(vp=cargs), _Arg(h=cppclass.handle)]
+    args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)]
     return _cdata_to_cobject(space, call_capi(space, 'call_o', args))
 
 def c_get_function_address(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return rffi.cast(C_FUNC_PTR,
         _cdata_to_ptr(space, call_capi(space, 'get_function_address', args)))
 
 # handling of function argument buffer ---------------------------------------
 def c_allocate_function_args(space, size):
-    return _cdata_to_ptr(space, call_capi(space, 'allocate_function_args', [_Arg(l=size)]))
+    return _cdata_to_ptr(space, call_capi(space, 'allocate_function_args', [_ArgL(size)]))
 def c_deallocate_function_args(space, cargs):
-    call_capi(space, 'deallocate_function_args', [_Arg(vp=cargs)])
+    call_capi(space, 'deallocate_function_args', [_ArgP(cargs)])
 def c_function_arg_sizeof(space):
     state = space.fromcache(State)
     return state.c_sizeof_farg
@@ -388,34 +413,34 @@
 
 # scope reflection information -----------------------------------------------
 def c_is_namespace(space, scope):
-    return space.bool_w(call_capi(space, 'is_namespace', [_Arg(h=scope)]))
+    return space.bool_w(call_capi(space, 'is_namespace', [_ArgH(scope)]))
 def c_is_template(space, name):
-    return space.bool_w(call_capi(space, 'is_template', [_Arg(s=name)]))
+    return space.bool_w(call_capi(space, 'is_template', [_ArgS(name)]))
 def c_is_abstract(space, cpptype):
-    return space.bool_w(call_capi(space, 'is_abstract', [_Arg(h=cpptype)]))
+    return space.bool_w(call_capi(space, 'is_abstract', [_ArgH(cpptype)]))
 def c_is_enum(space, name):
-    return space.bool_w(call_capi(space, 'is_enum', [_Arg(s=name)]))
+    return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)]))
 
 # type/class reflection information ------------------------------------------
 def c_final_name(space, cpptype):
-    return charp2str_free(space, call_capi(space, 'final_name', [_Arg(h=cpptype)]))
+    return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)]))
 def c_scoped_final_name(space, cpptype):
-    return charp2str_free(space, call_capi(space, 'scoped_final_name', [_Arg(h=cpptype)]))
+    return charp2str_free(space, call_capi(space, 'scoped_final_name', [_ArgH(cpptype)]))
 def c_has_complex_hierarchy(space, handle):
-    return space.bool_w(call_capi(space, 'has_complex_hierarchy', [_Arg(h=handle)]))
+    return space.bool_w(call_capi(space, 'has_complex_hierarchy', [_ArgH(handle)]))
 def c_num_bases(space, cppclass):
-    return space.int_w(call_capi(space, 'num_bases', [_Arg(h=cppclass.handle)]))
+    return space.int_w(call_capi(space, 'num_bases', [_ArgH(cppclass.handle)]))
 def c_base_name(space, cppclass, base_index):
-    args = [_Arg(h=cppclass.handle), _Arg(l=base_index)]
+    args = [_ArgH(cppclass.handle), _ArgL(base_index)]
     return charp2str_free(space, call_capi(space, 'base_name', args))
 def c_is_subtype(space, derived, base):
     jit.promote(base)
     if derived == base:
         return bool(1)
-    return space.bool_w(call_capi(space, 'is_subtype', [_Arg(h=derived.handle), _Arg(h=base.handle)]))
+    return space.bool_w(call_capi(space, 'is_subtype', [_ArgH(derived.handle), _ArgH(base.handle)]))
 
 def _c_base_offset(space, derived_h, base_h, address, direction):
-    args = [_Arg(h=derived_h), _Arg(h=base_h), _Arg(h=address), _Arg(l=direction)]
+    args = [_ArgH(derived_h), _ArgH(base_h), _ArgH(address), _ArgL(direction)]
     return _cdata_to_ptrdiff_t(space, call_capi(space, 'base_offset', args))
 def c_base_offset(space, derived, base, address, direction):
     if derived == base:
@@ -426,13 +451,13 @@
 
 # method/function reflection information -------------------------------------
 def c_num_methods(space, cppscope):
-    args = [_Arg(h=cppscope.handle)]
+    args = [_ArgH(cppscope.handle)]
     return space.int_w(call_capi(space, 'num_methods', args))
 def c_method_index_at(space, cppscope, imethod):
-    args = [_Arg(h=cppscope.handle), _Arg(l=imethod)]
+    args = [_ArgH(cppscope.handle), _ArgL(imethod)]
     return space.int_w(call_capi(space, 'method_index_at', args))
 def c_method_indices_from_name(space, cppscope, name):
-    args = [_Arg(h=cppscope.handle), _Arg(s=name)]
+    args = [_ArgH(cppscope.handle), _ArgS(name)]
     indices = rffi.cast(C_INDEX_ARRAY,
                         _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args)))
     if not indices:
@@ -448,91 +473,91 @@
     return py_indices
 
 def c_method_name(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return charp2str_free(space, call_capi(space, 'method_name', args))
 def c_method_result_type(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return charp2str_free(space, call_capi(space, 'method_result_type', args))
 def c_method_num_args(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.int_w(call_capi(space, 'method_num_args', args))
 def c_method_req_args(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.int_w(call_capi(space, 'method_req_args', args))
 def c_method_arg_type(space, cppscope, index, arg_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index), _Arg(l=arg_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(arg_index)]
     return charp2str_free(space, call_capi(space, 'method_arg_type', args))
 def c_method_arg_default(space, cppscope, index, arg_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index), _Arg(l=arg_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(arg_index)]
     return charp2str_free(space, call_capi(space, 'method_arg_default', args))
 def c_method_signature(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return charp2str_free(space, call_capi(space, 'method_signature', args))
 
 def c_method_is_template(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
 def _c_method_num_template_args(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.int_w(call_capi(space, 'method_num_template_args', args)) 
 def c_template_args(space, cppscope, index):
     nargs = _c_method_num_template_args(space, cppscope, index)
-    arg1 = _Arg(h=cppscope.handle)
-    arg2 = _Arg(l=index)
+    arg1 = _ArgH(cppscope.handle)
+    arg2 = _ArgL(index)
     args = [c_resolve_name(space, charp2str_free(space,
-                call_capi(space, 'method_template_arg_name', [arg1, arg2, _Arg(l=iarg)]))
+                call_capi(space, 'method_template_arg_name', [arg1, arg2, _ArgL(iarg)]))
             ) for iarg in range(nargs)]
     return args
 
 def c_get_method(space, cppscope, index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=index)]
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method', args)))
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
-        args = [_Arg(h=nss.handle), _Arg(h=lc.handle), _Arg(h=rc.handle), _Arg(s=op)]
+        args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
         return rffi.cast(WLAVC_INDEX, space.int_w(call_capi(space, 'get_global_operator', args)))
     return rffi.cast(WLAVC_INDEX, -1)
 
 # method properties ----------------------------------------------------------
 def c_is_constructor(space, cppclass, index):
-    args = [_Arg(h=cppclass.handle), _Arg(l=index)]
+    args = [_ArgH(cppclass.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'is_constructor', args))
 def c_is_staticmethod(space, cppclass, index):
-    args = [_Arg(h=cppclass.handle), _Arg(l=index)]
+    args = [_ArgH(cppclass.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'is_staticmethod', args))
 
 # data member reflection information -----------------------------------------
 def c_num_datamembers(space, cppscope):
-    return space.int_w(call_capi(space, 'num_datamembers', [_Arg(h=cppscope.handle)]))
+    return space.int_w(call_capi(space, 'num_datamembers', [_ArgH(cppscope.handle)]))
 def c_datamember_name(space, cppscope, datamember_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=datamember_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
     return charp2str_free(space, call_capi(space, 'datamember_name', args))
 def c_datamember_type(space, cppscope, datamember_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=datamember_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
     return  charp2str_free(space, call_capi(space, 'datamember_type', args))
 def c_datamember_offset(space, cppscope, datamember_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=datamember_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
     return _cdata_to_ptrdiff_t(space, call_capi(space, 'datamember_offset', args))
 
 def c_datamember_index(space, cppscope, name):
-    args = [_Arg(h=cppscope.handle), _Arg(s=name)]
+    args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.int_w(call_capi(space, 'datamember_index', args))
 
 # data member properties -----------------------------------------------------
 def c_is_publicdata(space, cppscope, datamember_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=datamember_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
     return space.bool_w(call_capi(space, 'is_publicdata', args))
 def c_is_staticdata(space, cppscope, datamember_index):
-    args = [_Arg(h=cppscope.handle), _Arg(l=datamember_index)]
+    args = [_ArgH(cppscope.handle), _ArgL(datamember_index)]
     return space.bool_w(call_capi(space, 'is_staticdata', args))
 
 # misc helpers ---------------------------------------------------------------
 def c_strtoll(space, svalue):
-    return space.r_longlong_w(call_capi(space, 'strtoll', [_Arg(s=svalue)]))
+    return space.r_longlong_w(call_capi(space, 'strtoll', [_ArgS(svalue)]))
 def c_strtoull(space, svalue):
-    return space.r_ulonglong_w(call_capi(space, 'strtoull', [_Arg(s=svalue)]))
+    return space.r_ulonglong_w(call_capi(space, 'strtoull', [_ArgS(svalue)]))
 def c_free(space, voidp):
-    call_capi(space, 'free', [_Arg(vp=voidp)])
+    call_capi(space, 'free', [_ArgP(voidp)])
 
 def charp2str_free(space, cdata):
     charp = rffi.cast(rffi.CCHARP, _cdata_to_ptr(space, cdata))
@@ -542,26 +567,26 @@
 
 def c_charp2stdstring(space, svalue, sz):
     return _cdata_to_cobject(space, call_capi(space, 'charp2stdstring',
-        [_Arg(s=svalue), _Arg(h=rffi.cast(rffi.ULONG, sz))]))
+        [_ArgS(svalue), _ArgH(rffi.cast(rffi.ULONG, sz))]))
 def c_stdstring2charp(space, cppstr):
     sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw')
     try:
         w_cstr = call_capi(space, 'stdstring2charp',
-            [_Arg(h=cppstr), _Arg(vp=rffi.cast(rffi.VOIDP, sz))])
+            [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))])
         cstr_len = intmask(sz[0])
     finally:
         lltype.free(sz, flavor='raw')
     return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len)
 def c_stdstring2stdstring(space, cppobject):
-    return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_Arg(h=cppobject)]))
+    return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
 def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_Arg(s=pystr)]))
+    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
 
 def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_Arg(s=pystr)]))
+    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
 def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_Arg(s=pystr)]))
+    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
 
 
 # TODO: factor these out ...

From pypy.commits at gmail.com  Wed Nov 23 15:01:05 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 12:01:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use default's implementation of
 PyMemoryView_FromBuffer
Message-ID: <5835f581.031f1c0a.b352b.9018@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r88592:f863a942eb67
Date: 2016-11-23 20:00 +0000
http://bitbucket.org/pypy/pypy/changeset/f863a942eb67/

Log:	Use default's implementation of PyMemoryView_FromBuffer

diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -112,12 +112,10 @@
     The memoryview object then owns the buffer represented by view, which
     means you shouldn't try to call PyBuffer_Release() yourself: it
     will be done on deallocation of the memoryview object."""
-    if not view.c_buf:
-        raise oefmt(space.w_ValueError,
-                    "cannot make memory view from a buffer with a NULL data "
-                    "pointer")
-    buf = CBuffer(space, view.c_buf, view.c_len, view.c_obj)
-    return space.wrap(W_MemoryView(buf))
+    w_obj = from_ref(space, view.c_obj)
+    if isinstance(w_obj, W_MemoryView):
+        return w_obj
+    return space.call_method(space.builtin, "memoryview", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):

From pypy.commits at gmail.com  Wed Nov 23 15:38:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 12:38:28 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: translation fix
Message-ID: <5835fe44.ca06c20a.4b3f6.cd5a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88593:1feae2bc5e13
Date: 2016-11-23 20:55 +0100
http://bitbucket.org/pypy/pypy/changeset/1feae2bc5e13/

Log:	translation fix

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1731,15 +1731,25 @@
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        if not is_value:
-            opnum = rop.COND_CALL
-        else:
-            opnum = OpHelpers.cond_call_value_for_descr(descr)
+        allboxes = [condbox] + allboxes
         # COND_CALL cannot be pure (=elidable): it has no result.
         # On the other hand, COND_CALL_VALUE is always calling a pure
         # function.
-        return self.execute_varargs(opnum, [condbox] + allboxes, descr,
-                                    exc, pure=is_value)
+        if not is_value:
+            return self.execute_varargs(rop.COND_CALL, allboxes, descr,
+                                        exc, pure=False)
+        else:
+            opnum = OpHelpers.cond_call_value_for_descr(descr)
+            # work around the fact that execute_varargs() wants a
+            # constant for first argument
+            if opnum == rop.COND_CALL_VALUE_I:
+                return self.execute_varargs(rop.COND_CALL_VALUE_I, allboxes,
+                                            descr, exc, pure=True)
+            elif opnum == rop.COND_CALL_VALUE_R:
+                return self.execute_varargs(rop.COND_CALL_VALUE_R, allboxes,
+                                            descr, exc, pure=True)
+            else:
+                raise AssertionError
 
     def _do_jit_force_virtual(self, allboxes, descr, pc):
         assert len(allboxes) == 2

From pypy.commits at gmail.com  Wed Nov 23 15:38:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 12:38:30 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: failing test
Message-ID: <5835fe46.581d1c0a.9dd87.9c91@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88594:72f9fe55b596
Date: 2016-11-23 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/72f9fe55b596/

Log:	failing test

diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -951,3 +951,10 @@
         self.meta_interp(f, [222, 3333])
         self.check_simple_loop({'guard_true': 1, 'int_add': 1,
                                 'int_lt': 1, 'jump': 1})
+
+    def test_string_hashing(self):
+        def f(i):
+            s = str(i)
+            d = {s: s + s}
+            return len(d[s])
+        assert self.interp_operations(f, [222]) == 6

From pypy.commits at gmail.com  Wed Nov 23 16:34:02 2016
From: pypy.commits at gmail.com (wlav)
Date: Wed, 23 Nov 2016 13:34:02 -0800 (PST)
Subject: [pypy-commit] pypy cling-support: coding convention and prepare for
 more test
Message-ID: <58360b4a.d32f1c0a.ea5b4.ace5@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cling-support
Changeset: r88595:cf4a6dabef21
Date: 2016-11-23 13:26 -0800
http://bitbucket.org/pypy/pypy/changeset/cf4a6dabef21/

Log:	coding convention and prepare for more test

diff --git a/pypy/module/cppyy/test/datatypes.cxx b/pypy/module/cppyy/test/datatypes.cxx
--- a/pypy/module/cppyy/test/datatypes.cxx
+++ b/pypy/module/cppyy/test/datatypes.cxx
@@ -2,23 +2,31 @@
 
 
 //===========================================================================
-cppyy_test_data::cppyy_test_data() : m_owns_arrays(false)
+std::vector<EFruit> vecFruits{kCitrus, kApple};
+
+
+//===========================================================================
+CppyyTestData::CppyyTestData() : m_owns_arrays(false)
 {
-    m_bool   = false;
-    m_char   = 'a';
-    m_uchar  = 'c';
-    m_short  = -11;
-    m_ushort =  11u;
-    m_int    = -22;
-    m_uint   =  22u;
-    m_long   = -33l;
-    m_ulong  =  33ul;
-    m_llong  = -44ll;
-    m_ullong =  55ull;
-    m_float  = -66.f;
-    m_double = -77.;
-    m_enum   = kNothing;
-    m_voidp  = (void*)0;
+    m_bool    = false;
+    m_char    = 'a';
+    m_schar   = 'b';
+    m_uchar   = 'c';
+    m_short   = -11;
+    m_ushort  =  11u;
+    m_int     = -22;
+    m_uint    =  22u;
+    m_long    = -33l;
+    m_ulong   =  33ul;
+    m_llong   = -44ll;
+    m_ullong  =  44ull;
+    m_long64  = -55ll;
+    m_ulong64 =  55ull;
+    m_float   = -66.f;
+    m_double  = -77.;
+    m_ldouble = -88.l;
+    m_enum    = kNothing;
+    m_voidp   = (void*)0;
 
     m_bool_array2   = new bool[N];
     m_short_array2  = new short[N];
@@ -61,12 +69,12 @@
     m_ppod = &m_pod;
 };
 
-cppyy_test_data::~cppyy_test_data()
+CppyyTestData::~CppyyTestData()
 {
     destroy_arrays();
 }
 
-void cppyy_test_data::destroy_arrays() {
+void CppyyTestData::destroy_arrays() {
     if (m_owns_arrays == true) {
         delete[] m_bool_array2;
         delete[] m_short_array2;
@@ -84,159 +92,244 @@
 }
 
 //- getters -----------------------------------------------------------------
-bool           cppyy_test_data::get_bool()   { return m_bool; }
-char           cppyy_test_data::get_char()   { return m_char; }
-unsigned char  cppyy_test_data::get_uchar()  { return m_uchar; }
-short          cppyy_test_data::get_short()  { return m_short; }
-unsigned short cppyy_test_data::get_ushort() { return m_ushort; }
-int            cppyy_test_data::get_int()    { return m_int; }
-unsigned int   cppyy_test_data::get_uint()   { return m_uint; }
-long           cppyy_test_data::get_long()   { return m_long; }
-unsigned long  cppyy_test_data::get_ulong()  { return m_ulong; }
-long long      cppyy_test_data::get_llong()  { return m_llong; }
-unsigned long long cppyy_test_data::get_ullong()  { return m_ullong; }
-float          cppyy_test_data::get_float()  { return m_float; }
-double         cppyy_test_data::get_double() { return m_double; }
-cppyy_test_data::what cppyy_test_data::get_enum() { return m_enum; }
-void*          cppyy_test_data::get_voidp()  { return m_voidp; }
+bool                 CppyyTestData::get_bool()    { return m_bool; }
+char                 CppyyTestData::get_char()    { return m_char; }
+signed char          CppyyTestData::get_schar()   { return m_schar; }
+unsigned char        CppyyTestData::get_uchar()   { return m_uchar; }
+short                CppyyTestData::get_short()   { return m_short; }
+unsigned short       CppyyTestData::get_ushort()  { return m_ushort; }
+int                  CppyyTestData::get_int()     { return m_int; }
+unsigned int         CppyyTestData::get_uint()    { return m_uint; }
+long                 CppyyTestData::get_long()    { return m_long; }
+unsigned long        CppyyTestData::get_ulong()   { return m_ulong; }
+long long            CppyyTestData::get_llong()   { return m_llong; }
+unsigned long long   CppyyTestData::get_ullong()  { return m_ullong; }
+Long64_t             CppyyTestData::get_long64()  { return m_long64; }
+ULong64_t            CppyyTestData::get_ulong64() { return m_ulong64; }
+float                CppyyTestData::get_float()   { return m_float; }
+double               CppyyTestData::get_double()  { return m_double; }
+long double          CppyyTestData::get_ldouble() { return m_ldouble; }
+CppyyTestData::EWhat CppyyTestData::get_enum()    { return m_enum; }
+void*                CppyyTestData::get_voidp()   { return m_voidp; }
 
-bool*           cppyy_test_data::get_bool_array()    { return m_bool_array; }
-bool*           cppyy_test_data::get_bool_array2()   { return m_bool_array2; }
-short*          cppyy_test_data::get_short_array()   { return m_short_array; }
-short*          cppyy_test_data::get_short_array2()  { return m_short_array2; }
-unsigned short* cppyy_test_data::get_ushort_array()  { return m_ushort_array; }
-unsigned short* cppyy_test_data::get_ushort_array2() { return m_ushort_array2; }
-int*            cppyy_test_data::get_int_array()     { return m_int_array; }
-int*            cppyy_test_data::get_int_array2()    { return m_int_array2; }
-unsigned int*   cppyy_test_data::get_uint_array()    { return m_uint_array; }
-unsigned int*   cppyy_test_data::get_uint_array2()   { return m_uint_array2; }
-long*           cppyy_test_data::get_long_array()    { return m_long_array; }
-long*           cppyy_test_data::get_long_array2()   { return m_long_array2; }
-unsigned long*  cppyy_test_data::get_ulong_array()   { return m_ulong_array; }
-unsigned long*  cppyy_test_data::get_ulong_array2()  { return m_ulong_array2; }
+bool*           CppyyTestData::get_bool_array()    { return m_bool_array; }
+bool*           CppyyTestData::get_bool_array2()   { return m_bool_array2; }
+short*          CppyyTestData::get_short_array()   { return m_short_array; }
+short*          CppyyTestData::get_short_array2()  { return m_short_array2; }
+unsigned short* CppyyTestData::get_ushort_array()  { return m_ushort_array; }
+unsigned short* CppyyTestData::get_ushort_array2() { return m_ushort_array2; }
+int*            CppyyTestData::get_int_array()     { return m_int_array; }
+int*            CppyyTestData::get_int_array2()    { return m_int_array2; }
+unsigned int*   CppyyTestData::get_uint_array()    { return m_uint_array; }
+unsigned int*   CppyyTestData::get_uint_array2()   { return m_uint_array2; }
+long*           CppyyTestData::get_long_array()    { return m_long_array; }
+long*           CppyyTestData::get_long_array2()   { return m_long_array2; }
+unsigned long*  CppyyTestData::get_ulong_array()   { return m_ulong_array; }
+unsigned long*  CppyyTestData::get_ulong_array2()  { return m_ulong_array2; }
 
-float*  cppyy_test_data::get_float_array()   { return m_float_array; }
-float*  cppyy_test_data::get_float_array2()  { return m_float_array2; }
-double* cppyy_test_data::get_double_array()  { return m_double_array; }
-double* cppyy_test_data::get_double_array2() { return m_double_array2; }
+float*  CppyyTestData::get_float_array()   { return m_float_array; }
+float*  CppyyTestData::get_float_array2()  { return m_float_array2; }
+double* CppyyTestData::get_double_array()  { return m_double_array; }
+double* CppyyTestData::get_double_array2() { return m_double_array2; }
 
-cppyy_test_pod cppyy_test_data::get_pod_val() { return m_pod; }
-cppyy_test_pod* cppyy_test_data::get_pod_val_ptr() { return &m_pod; }
-cppyy_test_pod& cppyy_test_data::get_pod_val_ref() { return m_pod; }
-cppyy_test_pod*& cppyy_test_data::get_pod_ptrref() { return m_ppod; }
+CppyyTestPod   CppyyTestData::get_pod_val()     { return m_pod; }
+CppyyTestPod*  CppyyTestData::get_pod_val_ptr() { return &m_pod; }
+CppyyTestPod&  CppyyTestData::get_pod_val_ref() { return m_pod; }
+CppyyTestPod*& CppyyTestData::get_pod_ptrref()  { return m_ppod; }
 
-cppyy_test_pod* cppyy_test_data::get_pod_ptr() { return m_ppod; }
+CppyyTestPod* CppyyTestData::get_pod_ptr() { return m_ppod; }
+
+//- getters const-ref -------------------------------------------------------
+const bool&                 CppyyTestData::get_bool_cr()    { return m_bool; }
+const char&                 CppyyTestData::get_char_cr()    { return m_char; }
+const signed char&          CppyyTestData::get_schar_cr()   { return m_schar; }
+const unsigned char&        CppyyTestData::get_uchar_cr()   { return m_uchar; }
+const short&                CppyyTestData::get_short_cr()   { return m_short; }
+const unsigned short&       CppyyTestData::get_ushort_cr()  { return m_ushort; }
+const int&                  CppyyTestData::get_int_cr()     { return m_int; }
+const unsigned int&         CppyyTestData::get_uint_cr()    { return m_uint; }
+const long&                 CppyyTestData::get_long_cr()    { return m_long; }
+const unsigned long&        CppyyTestData::get_ulong_cr()   { return m_ulong; }
+const long long&            CppyyTestData::get_llong_cr()   { return m_llong; }
+const unsigned long long&   CppyyTestData::get_ullong_cr()  { return m_ullong; }
+const Long64_t&             CppyyTestData::get_long64_cr()  { return m_long64; }
+const ULong64_t&            CppyyTestData::get_ulong64_cr() { return m_ulong64; }
+const float&                CppyyTestData::get_float_cr()   { return m_float; }
+const double&               CppyyTestData::get_double_cr()  { return m_double; }
+const long double&          CppyyTestData::get_ldouble_cr() { return m_ldouble; }
+const CppyyTestData::EWhat& CppyyTestData::get_enum_cr()    { return m_enum; }
+
+//- getters ref -------------------------------------------------------------
+bool&                 CppyyTestData::get_bool_r()    { return m_bool; }
+char&                 CppyyTestData::get_char_r()    { return m_char; }
+signed char&          CppyyTestData::get_schar_r()   { return m_schar; }
+unsigned char&        CppyyTestData::get_uchar_r()   { return m_uchar; }
+short&                CppyyTestData::get_short_r()   { return m_short; }
+unsigned short&       CppyyTestData::get_ushort_r()  { return m_ushort; }
+int&                  CppyyTestData::get_int_r()     { return m_int; }
+unsigned int&         CppyyTestData::get_uint_r()    { return m_uint; }
+long&                 CppyyTestData::get_long_r()    { return m_long; }
+unsigned long&        CppyyTestData::get_ulong_r()   { return m_ulong; }
+long long&            CppyyTestData::get_llong_r()   { return m_llong; }
+unsigned long long&   CppyyTestData::get_ullong_r()  { return m_ullong; }
+Long64_t&             CppyyTestData::get_long64_r()  { return m_long64; }
+ULong64_t&            CppyyTestData::get_ulong64_r() { return m_ulong64; }
+float&                CppyyTestData::get_float_r()   { return m_float; }
+double&               CppyyTestData::get_double_r()  { return m_double; }
+long double&          CppyyTestData::get_ldouble_r() { return m_ldouble; }
+CppyyTestData::EWhat& CppyyTestData::get_enum_r()    { return m_enum; }
 
 //- setters -----------------------------------------------------------------
-void cppyy_test_data::set_bool(bool b)                       { m_bool   = b; }
-void cppyy_test_data::set_char(char c)                       { m_char   = c; }
-void cppyy_test_data::set_uchar(unsigned char uc)            { m_uchar  = uc; }
-void cppyy_test_data::set_short(short s)                     { m_short  = s; }
-void cppyy_test_data::set_short_c(const short& s)            { m_short  = s; }
-void cppyy_test_data::set_ushort(unsigned short us)          { m_ushort = us; }
-void cppyy_test_data::set_ushort_c(const unsigned short& us) { m_ushort = us; }
-void cppyy_test_data::set_int(int i)                         { m_int    = i; }
-void cppyy_test_data::set_int_c(const int& i)                { m_int    = i; }
-void cppyy_test_data::set_uint(unsigned int ui)              { m_uint   = ui; }
-void cppyy_test_data::set_uint_c(const unsigned int& ui)     { m_uint   = ui; }
-void cppyy_test_data::set_long(long l)                       { m_long   = l; }
-void cppyy_test_data::set_long_c(const long& l)              { m_long   = l; }
-void cppyy_test_data::set_ulong(unsigned long ul)            { m_ulong  = ul; }
-void cppyy_test_data::set_ulong_c(const unsigned long& ul)   { m_ulong  = ul; }
-void cppyy_test_data::set_llong(long long ll)                { m_llong  = ll; }
-void cppyy_test_data::set_llong_c(const long long& ll)       { m_llong  = ll; }
-void cppyy_test_data::set_ullong(unsigned long long ull)     { m_ullong  = ull; }
-void cppyy_test_data::set_ullong_c(const unsigned long long& ull) { m_ullong  = ull; }
-void cppyy_test_data::set_float(float f)                     { m_float  = f; }
-void cppyy_test_data::set_float_c(const float& f)            { m_float  = f; }
-void cppyy_test_data::set_double(double d)                   { m_double = d; }
-void cppyy_test_data::set_double_c(const double& d)          { m_double = d; }
-void cppyy_test_data::set_enum(what w)                       { m_enum   = w; }
-void cppyy_test_data::set_voidp(void* p)                     { m_voidp  = p; }
+void CppyyTestData::set_bool(bool b)                       { m_bool    = b; }
+void CppyyTestData::set_char(char c)                       { m_char    = c; }
+void CppyyTestData::set_schar(signed char sc)              { m_schar   = sc; }
+void CppyyTestData::set_uchar(unsigned char uc)            { m_uchar   = uc; }
+void CppyyTestData::set_short(short s)                     { m_short   = s; }
+void CppyyTestData::set_ushort(unsigned short us)          { m_ushort  = us; }
+void CppyyTestData::set_int(int i)                         { m_int     = i; }
+void CppyyTestData::set_uint(unsigned int ui)              { m_uint    = ui; }
+void CppyyTestData::set_long(long l)                       { m_long    = l; }
+void CppyyTestData::set_ulong(unsigned long ul)            { m_ulong   = ul; }
+void CppyyTestData::set_llong(long long ll)                { m_llong   = ll; }
+void CppyyTestData::set_ullong(unsigned long long ull)     { m_ullong  = ull; }
+void CppyyTestData::set_long64(Long64_t l64)               { m_long64  = l64; }
+void CppyyTestData::set_ulong64(ULong64_t ul64)            { m_ulong64 = ul64; }
+void CppyyTestData::set_float(float f)                     { m_float   = f; }
+void CppyyTestData::set_double(double d)                   { m_double  = d; }
+void CppyyTestData::set_ldouble(long double ld)            { m_ldouble = ld; }
+void CppyyTestData::set_enum(EWhat w)                      { m_enum    = w; }
+void CppyyTestData::set_voidp(void* p)                     { m_voidp   = p; }
 
-void cppyy_test_data::set_pod_val(cppyy_test_pod p)            { m_pod = p; }
-void cppyy_test_data::set_pod_ptr_in(cppyy_test_pod* pp)       { m_pod = *pp; }
-void cppyy_test_data::set_pod_ptr_out(cppyy_test_pod* pp)      { *pp = m_pod; }
-void cppyy_test_data::set_pod_ref(const cppyy_test_pod& rp)    { m_pod = rp; }
-void cppyy_test_data::set_pod_ptrptr_in(cppyy_test_pod** ppp)  { m_pod = **ppp; }
-void cppyy_test_data::set_pod_void_ptrptr_in(void** pp)        { m_pod = **((cppyy_test_pod**)pp); }
-void cppyy_test_data::set_pod_ptrptr_out(cppyy_test_pod** ppp) { delete *ppp; *ppp = new cppyy_test_pod(m_pod); }
-void cppyy_test_data::set_pod_void_ptrptr_out(void** pp)       { delete *((cppyy_test_pod**)pp);
-                                                                 *((cppyy_test_pod**)pp) = new cppyy_test_pod(m_pod); }
+void CppyyTestData::set_pod_val(CppyyTestPod p)            { m_pod = p; }
+void CppyyTestData::set_pod_ptr_in(CppyyTestPod* pp)       { m_pod = *pp; }
+void CppyyTestData::set_pod_ptr_out(CppyyTestPod* pp)      { *pp = m_pod; }
+void CppyyTestData::set_pod_ref(const CppyyTestPod& rp)    { m_pod = rp; }
+void CppyyTestData::set_pod_ptrptr_in(CppyyTestPod** ppp)  { m_pod = **ppp; }
+void CppyyTestData::set_pod_void_ptrptr_in(void** pp)        { m_pod = **((CppyyTestPod**)pp); }
+void CppyyTestData::set_pod_ptrptr_out(CppyyTestPod** ppp) { delete *ppp; *ppp = new CppyyTestPod(m_pod); }
+void CppyyTestData::set_pod_void_ptrptr_out(void** pp)       { delete *((CppyyTestPod**)pp);
+                                                                 *((CppyyTestPod**)pp) = new CppyyTestPod(m_pod); }
 
-void cppyy_test_data::set_pod_ptr(cppyy_test_pod* pp)          { m_ppod = pp; }
+void CppyyTestData::set_pod_ptr(CppyyTestPod* pp)          { m_ppod = pp; }
+
+//- setters const-ref -------------------------------------------------------
+void CppyyTestData::set_bool_cr(const bool& b)                   { m_bool    = b; }
+void CppyyTestData::set_char_cr(const char& c)                   { m_char    = c; }
+void CppyyTestData::set_schar_cr(const signed char& sc)          { m_schar   = sc; }
+void CppyyTestData::set_uchar_cr(const unsigned char& uc)        { m_uchar   = uc; }
+void CppyyTestData::set_short_cr(const short& s)                 { m_short   = s; }
+void CppyyTestData::set_ushort_cr(const unsigned short& us)      { m_ushort  = us; }
+void CppyyTestData::set_int_cr(const int& i)                     { m_int     = i; }
+void CppyyTestData::set_uint_cr(const unsigned int& ui)          { m_uint    = ui; }
+void CppyyTestData::set_long_cr(const long& l)                   { m_long    = l; }
+void CppyyTestData::set_ulong_cr(const unsigned long& ul)        { m_ulong   = ul; }
+void CppyyTestData::set_llong_cr(const long long& ll)            { m_llong   = ll; }
+void CppyyTestData::set_ullong_cr(const unsigned long long& ull) { m_ullong  = ull; }
+void CppyyTestData::set_long64_cr(const Long64_t& l64)           { m_long64  = l64; }
+void CppyyTestData::set_ulong64_cr(const ULong64_t& ul64)        { m_ulong64 = ul64; }
+void CppyyTestData::set_float_cr(const float& f)                 { m_float   = f; }
+void CppyyTestData::set_double_cr(const double& d)               { m_double  = d; }
+void CppyyTestData::set_ldouble_cr(const long double& ld)        { m_ldouble = ld; }
+void CppyyTestData::set_enum_cr(const EWhat& w)                  { m_enum    = w; }
 
 //- passers -----------------------------------------------------------------
-short*          cppyy_test_data::pass_array(short* a)          { return a; }
-unsigned short* cppyy_test_data::pass_array(unsigned short* a) { return a; }
-int*            cppyy_test_data::pass_array(int* a)            { return a; }
-unsigned int*   cppyy_test_data::pass_array(unsigned int* a)   { return a; }
-long*           cppyy_test_data::pass_array(long* a)           { return a; }
-unsigned long*  cppyy_test_data::pass_array(unsigned long* a)  { return a; }
-float*          cppyy_test_data::pass_array(float* a)          { return a; }
-double*         cppyy_test_data::pass_array(double* a)         { return a; }
+short*          CppyyTestData::pass_array(short* a)          { return a; }
+unsigned short* CppyyTestData::pass_array(unsigned short* a) { return a; }
+int*            CppyyTestData::pass_array(int* a)            { return a; }
+unsigned int*   CppyyTestData::pass_array(unsigned int* a)   { return a; }
+long*           CppyyTestData::pass_array(long* a)           { return a; }
+unsigned long*  CppyyTestData::pass_array(unsigned long* a)  { return a; }
+float*          CppyyTestData::pass_array(float* a)          { return a; }
+double*         CppyyTestData::pass_array(double* a)         { return a; }
 
-char                cppyy_test_data::s_char   = 's';
-unsigned char       cppyy_test_data::s_uchar  = 'u';
-short               cppyy_test_data::s_short  = -101;
-unsigned short      cppyy_test_data::s_ushort =  255u;
-int                 cppyy_test_data::s_int    = -202;
-unsigned int        cppyy_test_data::s_uint   =  202u;
-long                cppyy_test_data::s_long   = -303l;
-unsigned long       cppyy_test_data::s_ulong  =  303ul;
-long long           cppyy_test_data::s_llong  = -404ll;
-unsigned long long  cppyy_test_data::s_ullong =  505ull;
-float               cppyy_test_data::s_float  = -606.f;
-double              cppyy_test_data::s_double = -707.;
-cppyy_test_data::what  cppyy_test_data::s_enum = cppyy_test_data::kNothing;
-void*               cppyy_test_data::s_voidp  = (void*)0;
+//- static data members -----------------------------------------------------
+bool                 CppyyTestData::s_bool    = false;
+char                 CppyyTestData::s_char    = 'c';
+signed char          CppyyTestData::s_schar   = 's';
+unsigned char        CppyyTestData::s_uchar   = 'u';
+short                CppyyTestData::s_short   = -101;
+unsigned short       CppyyTestData::s_ushort  =  255u;
+int                  CppyyTestData::s_int     = -202;
+unsigned int         CppyyTestData::s_uint    =  202u;
+long                 CppyyTestData::s_long    = -303l;
+unsigned long        CppyyTestData::s_ulong   =  303ul;
+long long            CppyyTestData::s_llong   = -404ll;
+unsigned long long   CppyyTestData::s_ullong  =  404ull;
+Long64_t             CppyyTestData::s_long64  = -505ll;
+ULong64_t            CppyyTestData::s_ulong64 = 505ull;
+float                CppyyTestData::s_float   = -606.f;
+double               CppyyTestData::s_double  = -707.;
+long double          CppyyTestData::s_ldouble = -808.l;
+CppyyTestData::EWhat CppyyTestData::s_enum    = CppyyTestData::kNothing;
+void*                CppyyTestData::s_voidp   = (void*)0;
 
 //- strings -----------------------------------------------------------------
-const char* cppyy_test_data::get_valid_string(const char* in) { return in; }
-const char* cppyy_test_data::get_invalid_string() { return (const char*)0; }
+const char* CppyyTestData::get_valid_string(const char* in) { return in; }
+const char* CppyyTestData::get_invalid_string() { return (const char*)0; }
 
 
 //= global functions ========================================================
-long get_pod_address(cppyy_test_data& c)
+long get_pod_address(CppyyTestData& c)
 {
     return (long)&c.m_pod;
 }
 
-long get_int_address(cppyy_test_data& c)
+long get_int_address(CppyyTestData& c)
 {
     return (long)&c.m_pod.m_int;
 }
 
-long get_double_address(cppyy_test_data& c)
+long get_double_address(CppyyTestData& c)
 {
     return (long)&c.m_pod.m_double;
 }
 
+
 //= global variables/pointers ===============================================
-int g_int = 42;
+bool               g_bool     = false;
+char               g_char     = 'w';
+signed char        g_schar    = 'v';
+unsigned char      g_uchar    = 'u';
+short              g_short    =  -88;
+unsigned short     g_ushort   =   88u;
+int                g_int      = -188;
+unsigned int       g_uint     =  188u;
+long               g_long     = -288;
+unsigned long      g_ulong    =  288ul;
+long long          g_llong    = -388ll;
+unsigned long long g_ullong   =  388ull;
+Long64_t           g_long64   = -488ll;
+ULong64_t          g_ulong64  =  488ull;
+float              g_float    = -588.f;
+double             g_double   = -688.;
+long double        g_ldouble  = -788.l;
+EFruit             g_enum     = kBanana;
+void*              g_voidp    = nullptr;
 
+
+//= global accessors ========================================================
 void set_global_int(int i) {
-   g_int = i;
+    g_int = i;
 }
 
 int get_global_int() {
-   return g_int;
+    return g_int;
 }
 
-cppyy_test_pod* g_pod = (cppyy_test_pod*)0;
+CppyyTestPod* g_pod = (CppyyTestPod*)0;
 
-bool is_global_pod(cppyy_test_pod* t) {
-   return t == g_pod;
+bool is_global_pod(CppyyTestPod* t) {
+    return t == g_pod;
 }
 
-void set_global_pod(cppyy_test_pod* t) {
-   g_pod = t;
+void set_global_pod(CppyyTestPod* t) {
+    g_pod = t;
 }
 
-cppyy_test_pod* get_global_pod() {
-   return g_pod;
+CppyyTestPod* get_global_pod() {
+    return g_pod;
 }
 
-cppyy_test_pod* get_null_pod() {
-   return (cppyy_test_pod*)0;
+CppyyTestPod* get_null_pod() {
+    return (CppyyTestPod*)0;
 }
diff --git a/pypy/module/cppyy/test/datatypes.h b/pypy/module/cppyy/test/datatypes.h
--- a/pypy/module/cppyy/test/datatypes.h
+++ b/pypy/module/cppyy/test/datatypes.h
@@ -1,23 +1,33 @@
+#include "RtypesCore.h"
+#include <vector>
+
 const int N = 5;
 
 
 //===========================================================================
-struct cppyy_test_pod {
+struct CppyyTestPod {
    int    m_int;
    double m_double;
 };
 
 
 //===========================================================================
-enum fruit { kApple=78, kBanana=29, kCitrus=34 };
+enum EFruit {kApple=78, kBanana=29, kCitrus=34};
+extern std::vector<EFruit> vecFruits;
 
 
 //===========================================================================
-class four_vector {
+namespace EnumSpace {
+   enum E {E1 = 1, E2};
+};
+
+
+//===========================================================================
+class FourVector {
 public:
-    four_vector(double x, double y, double z, double t) :
+    FourVector(double x, double y, double z, double t) :
         m_cc_called(false), m_x(x), m_y(y), m_z(z), m_t(t) {}
-    four_vector(const four_vector& s) :
+    FourVector(const FourVector& s) :
         m_cc_called(true), m_x(s.m_x), m_y(s.m_y), m_z(s.m_z), m_t(s.m_t) {}
 
     double operator[](int i) {
@@ -28,7 +38,7 @@
        return -1;
     }
 
-    bool operator==(const four_vector& o) {
+    bool operator==(const FourVector& o) {
        return (m_x == o.m_x && m_y == o.m_y &&
                m_z == o.m_z && m_t == o.m_t);
     }
@@ -42,13 +52,13 @@
 
 
 //===========================================================================
-class cppyy_test_data {
+class CppyyTestData {
 public:
-    cppyy_test_data();
-    ~cppyy_test_data();
+    CppyyTestData();
+    ~CppyyTestData();
 
 // special cases
-    enum what { kNothing=6, kSomething=111, kLots=42 };
+    enum EWhat { kNothing=6, kSomething=111, kLots=42 };
 
 // helper
     void destroy_arrays();
@@ -56,6 +66,7 @@
 // getters
     bool                 get_bool();
     char                 get_char();
+    signed char          get_schar();
     unsigned char        get_uchar();
     short                get_short();
     unsigned short       get_ushort();
@@ -65,9 +76,12 @@
     unsigned long        get_ulong();
     long long            get_llong();
     unsigned long long   get_ullong();
+    Long64_t             get_long64();
+    ULong64_t            get_ulong64();
     float                get_float();
     double               get_double();
-    what                 get_enum();
+    long double          get_ldouble();
+    EWhat                get_enum();
     void*                get_voidp();
 
     bool*           get_bool_array();
@@ -90,50 +104,104 @@
     double* get_double_array();
     double* get_double_array2();
 
-    cppyy_test_pod get_pod_val();                 // for m_pod
-    cppyy_test_pod* get_pod_val_ptr();
-    cppyy_test_pod& get_pod_val_ref();
-    cppyy_test_pod*& get_pod_ptrref();
+    CppyyTestPod get_pod_val();                 // for m_pod
+    CppyyTestPod* get_pod_val_ptr();
+    CppyyTestPod& get_pod_val_ref();
+    CppyyTestPod*& get_pod_ptrref();
 
-    cppyy_test_pod* get_pod_ptr();                // for m_ppod
+    CppyyTestPod* get_pod_ptr();                // for m_ppod
+
+// getters const-ref
+    const bool&               get_bool_cr();
+    const char&               get_char_cr();
+    const signed char&        get_schar_cr();
+    const unsigned char&      get_uchar_cr();
+    const short&              get_short_cr();
+    const unsigned short&     get_ushort_cr();
+    const int&                get_int_cr();
+    const unsigned int&       get_uint_cr();
+    const long&               get_long_cr();
+    const unsigned long&      get_ulong_cr();
+    const long long&          get_llong_cr();
+    const unsigned long long& get_ullong_cr();
+    const Long64_t&           get_long64_cr();
+    const ULong64_t&          get_ulong64_cr();
+    const float&              get_float_cr();
+    const double&             get_double_cr();
+    const long double&        get_ldouble_cr();
+    const EWhat&              get_enum_cr();
+
+// getters ref
+    bool&               get_bool_r();
+    char&               get_char_r();
+    signed char&        get_schar_r();
+    unsigned char&      get_uchar_r();
+    short&              get_short_r();
+    unsigned short&     get_ushort_r();
+    int&                get_int_r();
+    unsigned int&       get_uint_r();
+    long&               get_long_r();
+    unsigned long&      get_ulong_r();
+    long long&          get_llong_r();
+    unsigned long long& get_ullong_r();
+    Long64_t&           get_long64_r();
+    ULong64_t&          get_ulong64_r();
+    float&              get_float_r();
+    double&             get_double_r();
+    long double&        get_ldouble_r();
+    EWhat&              get_enum_r();
 
 // setters
-    void set_bool(bool b);
-    void set_char(char c);
-    void set_uchar(unsigned char uc);
-    void set_short(short s);
-    void set_short_c(const short& s);
-    void set_ushort(unsigned short us);
-    void set_ushort_c(const unsigned short& us);
-    void set_int(int i);
-    void set_int_c(const int& i);
-    void set_uint(unsigned int ui);
-    void set_uint_c(const unsigned int& ui);
-    void set_long(long l);
-    void set_long_c(const long& l);
-    void set_llong(long long ll);
-    void set_llong_c(const long long& ll);
-    void set_ulong(unsigned long ul);
-    void set_ulong_c(const unsigned long& ul);
-    void set_ullong(unsigned long long ll);
-    void set_ullong_c(const unsigned long long& ll);
-    void set_float(float f);
-    void set_float_c(const float& f);
-    void set_double(double d);
-    void set_double_c(const double& d);
-    void set_enum(what w);
-    void set_voidp(void* p);
+    void set_bool(bool);
+    void set_char(char);
+    void set_schar(signed char);
+    void set_uchar(unsigned char);
+    void set_short(short);
+    void set_ushort(unsigned short);
+    void set_int(int);
+    void set_uint(unsigned int);
+    void set_long(long);
+    void set_ulong(unsigned long);
+    void set_llong(long long);
+    void set_ullong(unsigned long long);
+    void set_long64(Long64_t);
+    void set_ulong64(ULong64_t);
+    void set_float(float);
+    void set_double(double);
+    void set_ldouble(long double);
+    void set_enum(EWhat);
+    void set_voidp(void*);
 
-    void set_pod_val(cppyy_test_pod);             // for m_pod
-    void set_pod_ptr_in(cppyy_test_pod*);
-    void set_pod_ptr_out(cppyy_test_pod*);
-    void set_pod_ref(const cppyy_test_pod&);
-    void set_pod_ptrptr_in(cppyy_test_pod**);
+    void set_pod_val(CppyyTestPod);             // for m_pod
+    void set_pod_ptr_in(CppyyTestPod*);
+    void set_pod_ptr_out(CppyyTestPod*);
+    void set_pod_ref(const CppyyTestPod&);
+    void set_pod_ptrptr_in(CppyyTestPod**);
     void set_pod_void_ptrptr_in(void**);
-    void set_pod_ptrptr_out(cppyy_test_pod**);
+    void set_pod_ptrptr_out(CppyyTestPod**);
     void set_pod_void_ptrptr_out(void**);
 
-    void set_pod_ptr(cppyy_test_pod*);            // for m_ppod
+    void set_pod_ptr(CppyyTestPod*);            // for m_ppod
+
+// setters const-ref
+    void set_bool_cr(const bool&);
+    void set_char_cr(const char&);
+    void set_schar_cr(const signed char&);
+    void set_uchar_cr(const unsigned char&);
+    void set_short_cr(const short&);
+    void set_ushort_cr(const unsigned short&);
+    void set_int_cr(const int&);
+    void set_uint_cr(const unsigned int&);
+    void set_long_cr(const long&);
+    void set_ulong_cr(const unsigned long&);
+    void set_llong_cr(const long long&);
+    void set_ullong_cr(const unsigned long long&);
+    void set_long64_cr(const Long64_t&);
+    void set_ulong64_cr(const ULong64_t&);
+    void set_float_cr(const float&);
+    void set_double_cr(const double&);
+    void set_ldouble_cr(const long double&);
+    void set_enum_cr(const EWhat&);
 
 // passers
     short*          pass_array(short*);
@@ -162,6 +230,7 @@
 // basic types
     bool                 m_bool;
     char                 m_char;
+    signed char          m_schar;
     unsigned char        m_uchar;
     short                m_short;
     unsigned short       m_ushort;
@@ -171,9 +240,12 @@
     unsigned long        m_ulong;
     long long            m_llong;
     unsigned long long   m_ullong;
+    Long64_t             m_long64;
+    ULong64_t            m_ulong64;
     float                m_float;
     double               m_double;
-    what                 m_enum;
+    long double          m_ldouble;
+    EWhat                m_enum;
     void*                m_voidp;
 
 // array types
@@ -191,18 +263,20 @@
     long*           m_long_array2;
     unsigned long   m_ulong_array[N];
     unsigned long*  m_ulong_array2;
- 
+
     float   m_float_array[N];
     float*  m_float_array2;
     double  m_double_array[N];
     double* m_double_array2;
 
 // object types
-    cppyy_test_pod m_pod;
-    cppyy_test_pod* m_ppod;
+    CppyyTestPod m_pod;
+    CppyyTestPod* m_ppod;
 
 public:
+    static bool                    s_bool;
     static char                    s_char;
+    static signed char             s_schar;
     static unsigned char           s_uchar;
     static short                   s_short;
     static unsigned short          s_ushort;
@@ -212,9 +286,12 @@
     static unsigned long           s_ulong;
     static long long               s_llong;
     static unsigned long long      s_ullong;
+    static Long64_t                s_long64;
+    static ULong64_t               s_ulong64;
     static float                   s_float;
     static double                  s_double;
-    static what                    s_enum;
+    static long double             s_ldouble;
+    static EWhat                   s_enum;
     static void*                   s_voidp;
 
 private:
@@ -223,18 +300,59 @@
 
 
 //= global functions ========================================================
-long get_pod_address(cppyy_test_data& c);
-long get_int_address(cppyy_test_data& c);
-long get_double_address(cppyy_test_data& c);
+long get_pod_address(CppyyTestData& c);
+long get_int_address(CppyyTestData& c);
+long get_double_address(CppyyTestData& c);
 
 
 //= global variables/pointers ===============================================
-extern int g_int;
+extern bool               g_bool;
+extern char               g_char;
+extern signed char        g_schar;
+extern unsigned char      g_uchar;
+extern short              g_short;
+extern unsigned short     g_ushort;
+extern int                g_int;
+extern unsigned int       g_uint;
+extern long               g_long;
+extern unsigned long      g_ulong;
+extern long long          g_llong;
+extern unsigned long long g_ullong;
+extern Long64_t           g_long64;
+extern ULong64_t          g_ulong64;
+extern float              g_float;
+extern double             g_double;
+extern long double        g_ldouble;
+extern EFruit             g_enum;
+extern void*              g_voidp;
+
+static const bool               g_c_bool    = true;
+static const char               g_c_char    = 'z';
+static const signed char        g_c_schar   = 'y';
+static const unsigned char      g_c_uchar   = 'x';
+static const short              g_c_short   =  -99;
+static const unsigned short     g_c_ushort  =   99u;
+static const int                g_c_int     = -199;
+static const unsigned int       g_c_uint    =  199u;
+static const long               g_c_long    = -299;
+static const unsigned long      g_c_ulong   =  299ul;
+static const long long          g_c_llong   = -399ll;
+static const unsigned long long g_c_ullong  =  399ull;
+static const Long64_t           g_c_long64  = -499ll;
+static const ULong64_t          g_c_ulong64 =  499ull;
+static const float              g_c_float   = -599.f;
+static const double             g_c_double  = -699.;
+static const long double        g_c_ldouble = -799.l;
+static const EFruit             g_c_enum    = kApple;
+static const void*              g_c_voidp   = nullptr;
+
+
+//= global accessors ========================================================
 void set_global_int(int i);
 int get_global_int();
 
-extern cppyy_test_pod* g_pod;
-bool is_global_pod(cppyy_test_pod* t);
-void set_global_pod(cppyy_test_pod* t);
-cppyy_test_pod* get_global_pod();
-cppyy_test_pod* get_null_pod();
+extern CppyyTestPod* g_pod;
+bool is_global_pod(CppyyTestPod* t);
+void set_global_pod(CppyyTestPod* t);
+CppyyTestPod* get_global_pod();
+CppyyTestPod* get_null_pod();
diff --git a/pypy/module/cppyy/test/datatypes.xml b/pypy/module/cppyy/test/datatypes.xml
--- a/pypy/module/cppyy/test/datatypes.xml
+++ b/pypy/module/cppyy/test/datatypes.xml
@@ -1,9 +1,9 @@
 <lcgdict>
 
-  <class pattern="cppyy_test_*" />
-  <class name="four_vector" />
+  <class pattern="CppyyTest*" />
+  <class name="FourVector" />
 
-  <enum name="fruit" />
+  <enum name="EFruit" />
 
   <function pattern="get_*" />
   <function pattern="set_*" />
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -31,10 +31,10 @@
         """Test read access to instance public data and verify values"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # reading boolean type
         assert c.m_bool == False
@@ -51,7 +51,7 @@
         assert c.m_long   == -33
         assert c.m_ulong  ==  33
         assert c.m_llong  == -44
-        assert c.m_ullong ==  55
+        assert c.m_ullong ==  44
 
         # reading floating point types
         assert round(c.m_float  + 66., 5) == 0
@@ -106,11 +106,11 @@
         raises(IndexError, c.m_double_array.__getitem__, self.N)
 
         # can not access an instance member on the class
-        raises(ReferenceError, getattr, cppyy_test_data, 'm_bool')
-        raises(ReferenceError, getattr, cppyy_test_data, 'm_int')
+        raises(ReferenceError, getattr, CppyyTestData, 'm_bool')
+        raises(ReferenceError, getattr, CppyyTestData, 'm_int')
 
-        assert not hasattr(cppyy_test_data, 'm_bool')
-        assert not hasattr(cppyy_test_data, 'm_int')
+        assert not hasattr(CppyyTestData, 'm_bool')
+        assert not hasattr(CppyyTestData, 'm_int')
 
         c.destruct()
 
@@ -118,10 +118,10 @@
         """Test write access to instance public data and verify values"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # boolean types through functions
         c.set_bool(True);  assert c.get_bool() == True
@@ -165,7 +165,7 @@
             assert eval('c.m_%s' % names[i]) == 2*i
 
         for i in range(len(names)):
-            exec 'c.set_%s_c(%d)' % (names[i],3*i)
+            exec 'c.set_%s_cr(%d)' % (names[i],3*i)
             assert eval('c.m_%s' % names[i]) == 3*i
 
         # float types through functions
@@ -173,12 +173,12 @@
         c.set_double( 0.456 ); assert round(c.get_double() - 0.456, 8) == 0
 
         # float types through data members
-        c.m_float = 0.123;     assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_float(0.234);    assert round(c.m_float      - 0.234, 5) == 0
-        c.set_float_c(0.456);  assert round(c.m_float      - 0.456, 5) == 0
-        c.m_double = 0.678;    assert round(c.get_double() - 0.678, 8) == 0
-        c.set_double(0.890);   assert round(c.m_double     - 0.890, 8) == 0
-        c.set_double_c(0.012); assert round(c.m_double     - 0.012, 8) == 0
+        c.m_float = 0.123;      assert round(c.get_float()  - 0.123, 5) == 0
+        c.set_float(0.234);     assert round(c.m_float      - 0.234, 5) == 0
+        c.set_float_cr(0.456);  assert round(c.m_float      - 0.456, 5) == 0
+        c.m_double = 0.678;     assert round(c.get_double() - 0.678, 8) == 0
+        c.set_double(0.890);    assert round(c.m_double     - 0.890, 8) == 0
+        c.set_double_cr(0.012); assert round(c.m_double     - 0.012, 8) == 0
 
         # arrays; there will be pointer copies, so destroy the current ones
         c.destroy_arrays()
@@ -205,10 +205,10 @@
         """Test passing of array arguments"""
 
         import cppyy, array, sys
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         a = range(self.N)
         # test arrays in mixed order, to give overload resolution a workout
@@ -243,39 +243,39 @@
         """Test read access to class public data and verify values"""
 
         import cppyy, sys
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # char types
-        assert cppyy_test_data.s_char  == 's'
-        assert c.s_char                == 's'
+        assert CppyyTestData.s_char    == 'c'
+        assert c.s_char                == 'c'
         assert c.s_uchar               == 'u'
-        assert cppyy_test_data.s_uchar == 'u'
+        assert CppyyTestData.s_uchar   == 'u'
 
         # integer types
-        assert cppyy_test_data.s_short  == -101
+        assert CppyyTestData.s_short    == -101
         assert c.s_short                == -101
         assert c.s_ushort               ==  255
-        assert cppyy_test_data.s_ushort ==  255
-        assert cppyy_test_data.s_int    == -202
+        assert CppyyTestData.s_ushort   ==  255
+        assert CppyyTestData.s_int      == -202
         assert c.s_int                  == -202
         assert c.s_uint                 ==  202
-        assert cppyy_test_data.s_uint   ==  202
-        assert cppyy_test_data.s_long   == -303L
+        assert CppyyTestData.s_uint     ==  202
+        assert CppyyTestData.s_long     == -303L
         assert c.s_long                 == -303L
         assert c.s_ulong                ==  303L
-        assert cppyy_test_data.s_ulong  ==  303L
-        assert cppyy_test_data.s_llong  == -404L
+        assert CppyyTestData.s_ulong    ==  303L
+        assert CppyyTestData.s_llong    == -404L
         assert c.s_llong                == -404L
-        assert c.s_ullong               ==  505L
-        assert cppyy_test_data.s_ullong ==  505L
+        assert c.s_ullong               ==  404L
+        assert CppyyTestData.s_ullong   ==  404L
 
         # floating point types
-        assert round(cppyy_test_data.s_float  + 606., 5) == 0
+        assert round(CppyyTestData.s_float  + 606., 5)   == 0
         assert round(c.s_float                + 606., 5) == 0
-        assert round(cppyy_test_data.s_double + 707., 8) == 0
+        assert round(CppyyTestData.s_double + 707., 8)   == 0
         assert round(c.s_double               + 707., 8) == 0
 
         c.destruct()
@@ -284,62 +284,62 @@
         """Test write access to class public data and verify values"""
 
         import cppyy, sys
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # char types
-        cppyy_test_data.s_char          = 'a'
+        CppyyTestData.s_char          = 'a'
         assert c.s_char                == 'a'
         c.s_char                        = 'b'
-        assert cppyy_test_data.s_char  == 'b'
-        cppyy_test_data.s_uchar         = 'c'
+        assert CppyyTestData.s_char  == 'b'
+        CppyyTestData.s_uchar         = 'c'
         assert c.s_uchar               == 'c'
         c.s_uchar                       = 'd'
-        assert cppyy_test_data.s_uchar == 'd'
-        raises(ValueError, setattr, cppyy_test_data, 's_uchar', -1)
+        assert CppyyTestData.s_uchar == 'd'
+        raises(ValueError, setattr, CppyyTestData, 's_uchar', -1)
         raises(ValueError, setattr, c,               's_uchar', -1)
 
         # integer types
         c.s_short                        = -102
-        assert cppyy_test_data.s_short  == -102
-        cppyy_test_data.s_short          = -203
+        assert CppyyTestData.s_short  == -102
+        CppyyTestData.s_short          = -203
         assert c.s_short                == -203
         c.s_ushort                       =  127
-        assert cppyy_test_data.s_ushort ==  127
-        cppyy_test_data.s_ushort         =  227
+        assert CppyyTestData.s_ushort ==  127
+        CppyyTestData.s_ushort         =  227
         assert c.s_ushort               ==  227
-        cppyy_test_data.s_int            = -234
+        CppyyTestData.s_int            = -234
         assert c.s_int                  == -234
         c.s_int                          = -321
-        assert cppyy_test_data.s_int    == -321
-        cppyy_test_data.s_uint           = 1234
+        assert CppyyTestData.s_int    == -321
+        CppyyTestData.s_uint           = 1234
         assert c.s_uint                 == 1234
         c.s_uint                         = 4321
-        assert cppyy_test_data.s_uint   == 4321
+        assert CppyyTestData.s_uint   == 4321
         raises(ValueError, setattr, c,               's_uint', -1)
-        raises(ValueError, setattr, cppyy_test_data, 's_uint', -1)
-        cppyy_test_data.s_long           = -87L
+        raises(ValueError, setattr, CppyyTestData, 's_uint', -1)
+        CppyyTestData.s_long           = -87L
         assert c.s_long                 == -87L
         c.s_long                         = 876L
-        assert cppyy_test_data.s_long   == 876L
-        cppyy_test_data.s_ulong          = 876L
+        assert CppyyTestData.s_long   == 876L
+        CppyyTestData.s_ulong          = 876L
         assert c.s_ulong                == 876L
         c.s_ulong                        = 678L
-        assert cppyy_test_data.s_ulong  == 678L
-        raises(ValueError, setattr, cppyy_test_data, 's_ulong', -1)
+        assert CppyyTestData.s_ulong  == 678L
+        raises(ValueError, setattr, CppyyTestData, 's_ulong', -1)
         raises(ValueError, setattr, c,               's_ulong', -1)
 
         # floating point types
-        cppyy_test_data.s_float                    = -3.1415
+        CppyyTestData.s_float                    = -3.1415
         assert round(c.s_float, 5 )               == -3.1415
         c.s_float                                  =  3.1415
-        assert round(cppyy_test_data.s_float, 5 ) ==  3.1415
+        assert round(CppyyTestData.s_float, 5 ) ==  3.1415
         import math
         c.s_double                                 = -math.pi
-        assert cppyy_test_data.s_double           == -math.pi
-        cppyy_test_data.s_double                   =  math.pi
+        assert CppyyTestData.s_double           == -math.pi
+        CppyyTestData.s_double                   =  math.pi
         assert c.s_double                         ==  math.pi
 
         c.destruct()
@@ -348,10 +348,10 @@
         """Test the ranges of integer types"""
 
         import cppyy, sys
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # TODO: should these be TypeErrors, or should char/bool raise
         #       ValueErrors? In any case, consistency is needed ...
@@ -364,10 +364,10 @@
         """Test conversions between builtin types"""
 
         import cppyy, sys
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         c.m_double = -1
         assert round(c.m_double + 1.0, 8) == 0
@@ -402,7 +402,7 @@
 
         raises(ReferenceError, 'gbl.g_pod.m_int')
 
-        c = gbl.cppyy_test_pod()
+        c = gbl.CppyyTestPod()
         c.m_int = 42
         c.m_double = 3.14
 
@@ -416,7 +416,7 @@
         assert c == d
         assert id(c) == id(d)
 
-        e = gbl.cppyy_test_pod()
+        e = gbl.CppyyTestPod()
         e.m_int = 43
         e.m_double = 2.14
 
@@ -431,44 +431,44 @@
         import cppyy
         gbl = cppyy.gbl
 
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         # test that the enum is accessible as a type
-        assert cppyy_test_data.what
+        assert CppyyTestData.EWhat
 
-        assert cppyy_test_data.kNothing   ==   6
-        assert cppyy_test_data.kSomething == 111
-        assert cppyy_test_data.kLots      ==  42
+        assert CppyyTestData.kNothing   ==   6
+        assert CppyyTestData.kSomething == 111
+        assert CppyyTestData.kLots      ==  42
 
-        assert cppyy_test_data.what(cppyy_test_data.kNothing) == cppyy_test_data.kNothing
-        assert cppyy_test_data.what(6) == cppyy_test_data.kNothing
+        assert CppyyTestData.EWhat(CppyyTestData.kNothing) == CppyyTestData.kNothing
+        assert CppyyTestData.EWhat(6) == CppyyTestData.kNothing
         # TODO: only allow instantiations with correct values (C++11)
 
-        assert c.get_enum() == cppyy_test_data.kNothing
-        assert c.m_enum == cppyy_test_data.kNothing
+        assert c.get_enum() == CppyyTestData.kNothing
+        assert c.m_enum == CppyyTestData.kNothing
 
-        c.m_enum = cppyy_test_data.kSomething
-        assert c.get_enum() == cppyy_test_data.kSomething
-        assert c.m_enum == cppyy_test_data.kSomething
+        c.m_enum = CppyyTestData.kSomething
+        assert c.get_enum() == CppyyTestData.kSomething
+        assert c.m_enum == CppyyTestData.kSomething
 
-        c.set_enum(cppyy_test_data.kLots)
-        assert c.get_enum() == cppyy_test_data.kLots
-        assert c.m_enum == cppyy_test_data.kLots
+        c.set_enum(CppyyTestData.kLots)
+        assert c.get_enum() == CppyyTestData.kLots
+        assert c.m_enum == CppyyTestData.kLots
 
-        assert c.s_enum == cppyy_test_data.s_enum
-        assert c.s_enum == cppyy_test_data.kNothing
-        assert cppyy_test_data.s_enum == cppyy_test_data.kNothing
+        assert c.s_enum == CppyyTestData.s_enum
+        assert c.s_enum == CppyyTestData.kNothing
+        assert CppyyTestData.s_enum == CppyyTestData.kNothing
 
-        c.s_enum = cppyy_test_data.kSomething
-        assert c.s_enum == cppyy_test_data.s_enum
-        assert c.s_enum == cppyy_test_data.kSomething
-        assert cppyy_test_data.s_enum == cppyy_test_data.kSomething
+        c.s_enum = CppyyTestData.kSomething
+        assert c.s_enum == CppyyTestData.s_enum
+        assert c.s_enum == CppyyTestData.kSomething
+        assert CppyyTestData.s_enum == CppyyTestData.kSomething
 
         # global enums
-        assert gbl.fruit          # test type accessible
+        assert gbl.EFruit          # test type accessible
         assert gbl.kApple  == 78
         assert gbl.kBanana == 29
         assert gbl.kCitrus == 34
@@ -477,9 +477,9 @@
         """Test passing/returning of a const char*"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
+        c = CppyyTestData()
         assert c.get_valid_string('aap') == 'aap'
         #assert c.get_invalid_string() == ''
 
@@ -487,11 +487,11 @@
         """Test copy constructor"""
 
         import cppyy
-        four_vector = cppyy.gbl.four_vector
+        FourVector = cppyy.gbl.FourVector
 
-        t1 = four_vector(1., 2., 3., -4.)
-        t2 = four_vector(0., 0., 0.,  0.)
-        t3 = four_vector(t1)
+        t1 = FourVector(1., 2., 3., -4.)
+        t2 = FourVector(0., 0., 0.,  0.)
+        t3 = FourVector(t1)
 
         assert t1 == t3
         assert t1 != t2
@@ -504,7 +504,7 @@
 
         import cppyy
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
 
         assert c.m_pod.m_int == 888
         assert c.m_pod.m_double == 3.14
@@ -531,11 +531,11 @@
 
         import cppyy
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         assert c.m_pod.m_int == 888
         assert c.m_pod.m_double == 3.14
 
-        p = cppyy.gbl.cppyy_test_pod()
+        p = cppyy.gbl.CppyyTestPod()
         p.m_int = 123
         assert p.m_int == 123
         p.m_double = 321.
@@ -545,12 +545,12 @@
         assert c.m_pod.m_int == 123
         assert c.m_pod.m_double == 321.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_ptr_in(p)
         assert c.m_pod.m_int == 123
         assert c.m_pod.m_double == 321.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_ptr_out(p)
         assert p.m_int == 888
         assert p.m_double == 3.14
@@ -558,26 +558,26 @@
         p.m_int = 555
         p.m_double = 666.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_ref(p)
         assert c.m_pod.m_int == 555
         assert c.m_pod.m_double == 666.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_ptrptr_in(p)
         assert c.m_pod.m_int == 555
         assert c.m_pod.m_double == 666.
         assert p.m_int == 555
         assert p.m_double == 666.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_void_ptrptr_in(p)
         assert c.m_pod.m_int == 555
         assert c.m_pod.m_double == 666.
         assert p.m_int == 555
         assert p.m_double == 666.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_ptrptr_out(p)
         assert c.m_pod.m_int == 888
         assert c.m_pod.m_double == 3.14
@@ -587,7 +587,7 @@
         p.m_int = 777
         p.m_double = 888.
 
-        c = cppyy.gbl.cppyy_test_data()
+        c = cppyy.gbl.CppyyTestData()
         c.set_pod_void_ptrptr_out(p)
         assert c.m_pod.m_int == 888
         assert c.m_pod.m_double == 3.14
@@ -600,7 +600,7 @@
         import cppyy
 
         for o in (0, None):
-            c = cppyy.gbl.cppyy_test_data()
+            c = cppyy.gbl.CppyyTestData()
             assert c.m_pod.m_int == 888
             assert c.m_pod.m_double == 3.14
             assert not not c.m_ppod
@@ -613,10 +613,10 @@
         """Test that privacy settings are respected"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
-        assert isinstance(c, cppyy_test_data)
+        c = CppyyTestData()
+        assert isinstance(c, CppyyTestData)
 
         raises(AttributeError, getattr, c, 'm_owns_arrays')
 
@@ -628,29 +628,29 @@
         import cppyy
         gbl = cppyy.gbl
 
-        c1 = cppyy.bind_object(0, gbl.cppyy_test_data)
+        c1 = cppyy.bind_object(0, gbl.CppyyTestData)
         assert c1 == None
         assert None == c1
 
-        c2 = cppyy.bind_object(0, gbl.cppyy_test_data)
+        c2 = cppyy.bind_object(0, gbl.CppyyTestData)
         assert c1 == c2
         assert c2 == c1
 
-        # four_vector overrides operator==
-        l1 = cppyy.bind_object(0, gbl.four_vector)
+        # FourVector overrides operator==
+        l1 = cppyy.bind_object(0, gbl.FourVector)
         assert l1 == None
         assert None == l1
 
         assert c1 != l1
         assert l1 != c1
 
-        l2 = cppyy.bind_object(0, gbl.four_vector)
+        l2 = cppyy.bind_object(0, gbl.FourVector)
         assert l1 == l2
         assert l2 == l1
 
-        l3 = gbl.four_vector(1, 2, 3, 4)
-        l4 = gbl.four_vector(1, 2, 3, 4)
-        l5 = gbl.four_vector(4, 3, 2, 1)
+        l3 = gbl.FourVector(1, 2, 3, 4)
+        l4 = gbl.FourVector(1, 2, 3, 4)
+        l5 = gbl.FourVector(4, 3, 2, 1)
         assert l3 == l4
         assert l4 == l3
 
@@ -664,7 +664,7 @@
 
         from cppyy import gbl
 
-        d = gbl.cppyy_test_pod()
+        d = gbl.CppyyTestPod()
 
         assert d
         assert not not d
@@ -677,9 +677,9 @@
         """Test usage of buffer sizing"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
+        c = CppyyTestData()
         for func in ['get_bool_array',   'get_bool_array2',
                      'get_ushort_array', 'get_ushort_array2',
                      'get_int_array',    'get_int_array2',
@@ -698,19 +698,19 @@
         """Test usage of void* data"""
 
         import cppyy
-        cppyy_test_data = cppyy.gbl.cppyy_test_data
+        CppyyTestData = cppyy.gbl.CppyyTestData
 
-        c = cppyy_test_data()
+        c = CppyyTestData()
 
         assert not cppyy.gbl.nullptr
 
         assert c.s_voidp                is cppyy.gbl.nullptr
-        assert cppyy_test_data.s_voidp  is cppyy.gbl.nullptr
+        assert CppyyTestData.s_voidp  is cppyy.gbl.nullptr
 
         assert c.m_voidp                is cppyy.gbl.nullptr
         assert c.get_voidp()            is cppyy.gbl.nullptr
 
-        c2 = cppyy_test_data()
+        c2 = CppyyTestData()
         assert c2.m_voidp               is cppyy.gbl.nullptr
         c.set_voidp(c2.m_voidp)
         assert c.m_voidp                is cppyy.gbl.nullptr
@@ -722,9 +722,9 @@
         c.set_voidp(c2)
         def address_equality_test(a, b):
             assert cppyy.addressof(a) == cppyy.addressof(b)
-            b2 = cppyy.bind_object(a, cppyy_test_data)
+            b2 = cppyy.bind_object(a, CppyyTestData)
             assert b is b2    # memory regulator recycles
-            b3 = cppyy.bind_object(cppyy.addressof(a), cppyy_test_data)
+            b3 = cppyy.bind_object(cppyy.addressof(a), CppyyTestData)
             assert b is b3    # likewise
 
         address_equality_test(c.m_voidp, c2)

From pypy.commits at gmail.com  Wed Nov 23 17:05:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 14:05:50 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Simplest solution to
 avoid troubles with a getfield on the hash: add a
Message-ID: <583612be.05371c0a.27ae6.b7ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88596:7e9c96bc7e3e
Date: 2016-11-23 22:43 +0100
http://bitbucket.org/pypy/pypy/changeset/7e9c96bc7e3e/

Log:	Simplest solution to avoid troubles with a getfield on the hash: add
	a custom operation

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -788,6 +788,9 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        return s._obj.container.hash
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -810,6 +813,9 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        return s._obj.container.hash
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -789,12 +789,20 @@
                                                 arrayfielddescr,
                                                 arraydescr)
             return []
+        # check for the string or unicode hash field
+        STRUCT = v_inst.concretetype.TO
+        if STRUCT == rstr.STR:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('strhash', [v_inst], op.result)
+        elif STRUCT == rstr.UNICODE:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('unicodehash', [v_inst], op.result)
         # check for _immutable_fields_ hints
-        immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+        immut = STRUCT._immutable_field(c_fieldname.value)
         need_live = False
         if immut:
             if (self.callcontrol is not None and
-                self.callcontrol.could_be_green_field(v_inst.concretetype.TO,
+                self.callcontrol.could_be_green_field(STRUCT,
                                                       c_fieldname.value)):
                 pure = '_greenfield'
                 need_live = True
@@ -802,10 +810,9 @@
                 pure = '_pure'
         else:
             pure = ''
-        self.check_field_access(v_inst.concretetype.TO)
-        argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
-        descr = self.cpu.fielddescrof(v_inst.concretetype.TO,
-                                      c_fieldname.value)
+        self.check_field_access(STRUCT)
+        argname = getattr(STRUCT, '_gckind', 'gc')
+        descr = self.cpu.fielddescrof(STRUCT, c_fieldname.value)
         kind = getkind(RESULT)[0]
         if argname != 'gc':
             assert argname == 'raw'
@@ -822,7 +829,7 @@
         if immut in (IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY):
             op1.opname += "_pure"
             descr1 = self.cpu.fielddescrof(
-                v_inst.concretetype.TO,
+                STRUCT,
                 quasiimmut.get_mutate_field_name(c_fieldname.value))
             return [SpaceOperation('-live-', [], None),
                    SpaceOperation('record_quasiimmut_field',
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1507,6 +1507,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_strhash(cpu, string):
+        return cpu.bh_strhash(string)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1523,6 +1526,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_unicodehash(cpu, unicode):
+        return cpu.bh_unicodehash(unicode)
 
     @arguments("i", "i")
     def bhimpl_rvmprof_code(leaving, unique_id):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1163,6 +1163,14 @@
         self.execute(rop.UNICODESETITEM, unicodebox, indexbox, newcharbox)
 
     @arguments("box")
+    def opimpl_strhash(self, strbox):
+        return self.execute(rop.STRHASH, strbox)
+
+    @arguments("box")
+    def opimpl_unicodehash(self, unicodebox):
+        return self.execute(rop.UNICODEHASH, unicodebox)
+
+    @arguments("box")
     def opimpl_newstr(self, lengthbox):
         return self.execute(rop.NEWSTR, lengthbox)
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1097,6 +1097,8 @@
     '_MALLOC_LAST',
     'FORCE_TOKEN/0/r',    # historical name; nowadays, returns the jitframe
     'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
+    'STRHASH/1/i',        # only reading the .hash field, might be zero so far
+    'UNICODEHASH/1/i',    #  "
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----

From pypy.commits at gmail.com  Wed Nov 23 17:05:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 14:05:54 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: fix some tests
Message-ID: <583612c2.c64bc20a.a1c99.eee5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88598:1323559e78f2
Date: 2016-11-23 23:05 +0100
http://bitbucket.org/pypy/pypy/changeset/1323559e78f2/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -194,7 +194,8 @@
                            'guard_true': 4, 'jump': 1,
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
-                           'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
+                           'call_i': 4, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
+                           'cond_call_value_i': 2, 'strhash': 4,
                            'guard_no_exception': 8, 'new': 2,
                            'guard_nonnull': 2})
 
@@ -273,7 +274,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call_i=4, call_n=1)
+        self.check_simple_loop(call_i=3, cond_call_value_i=1, call_n=1)
 
     def test_dict_array_write_invalidates_caches(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -295,7 +296,7 @@
         exp = f(10)
         res = self.meta_interp(f, [10])
         assert res == exp
-        self.check_simple_loop(call_i=5, call_n=2)
+        self.check_simple_loop(call_i=4, cond_call_value_i=1, call_n=2)
 
     def test_dict_double_lookup_2(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -314,7 +315,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call_i=2, call_n=1)
+        self.check_simple_loop(call_i=1, cond_call_value_i=1, call_n=1)
 
     def test_dict_eq_can_release_gil(self):
         from rpython.rtyper.lltypesystem import lltype, rffi

From pypy.commits at gmail.com  Wed Nov 23 17:05:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 Nov 2016 14:05:52 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: rewrite
 str/unicodehash, and add bh_str/unicodehash to llmodel
Message-ID: <583612c0.90a81c0a.b1ffa.be4d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88597:795c9b5e8990
Date: 2016-11-23 22:53 +0100
http://bitbucket.org/pypy/pypy/changeset/795c9b5e8990/

Log:	rewrite str/unicodehash, and add bh_str/unicodehash to llmodel

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -663,6 +663,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return s.hash
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return u.hash
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,7 +9,7 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
      FLAG_POINTER, CallDescr
 from rpython.jit.metainterp.history import JitCellToken
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1238,6 +1238,10 @@
                        'i3 = gc_load_i(p0,'
                                  '%(strlendescr.offset)s,'
                                  '%(strlendescr.field_size)s)'],
+        [True,  (1,),  'i3 = strhash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(strhashdescr.offset)s,'
+                                 '-%(strhashdescr.field_size)s)'],
         #[False, (1,),  'i3 = unicodelen(p0)' '->'
         #               'i3 = gc_load_i(p0,'
         #                       '%(unicodelendescr.offset)s,'
@@ -1246,7 +1250,10 @@
                        'i3 = gc_load_i(p0,'
                                '%(unicodelendescr.offset)s,'
                                '%(unicodelendescr.field_size)s)'],
-
+        [True,  (1,),  'i3 = unicodehash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(unicodehashdescr.offset)s,'
+                                 '-%(unicodehashdescr.field_size)s)'],
         ## getitem str/unicode
         [True,  (2,4), 'i3 = unicodegetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,'

From pypy.commits at gmail.com  Wed Nov 23 21:02:12 2016
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 23 Nov 2016 18:02:12 -0800 (PST)
Subject: [pypy-commit] pypy default: Add translation option to keep
 annotating as much as possible when
Message-ID: <58364a24.8f95c20a.2b130.2517@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88599:cbf3627becf5
Date: 2016-11-23 23:10 +0000
http://bitbucket.org/pypy/pypy/changeset/cbf3627becf5/

Log:	Add translation option to keep annotating as much as possible when
	AnnotationErrors are raised, and report all these errors at the end.

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -22,7 +22,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -50,6 +51,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -202,6 +206,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -348,6 +358,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -392,6 +404,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -482,6 +498,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -67,7 +67,8 @@
         if self.annotator is not None:
             raise ValueError("we already have an annotator")
         from rpython.annotator.annrpython import RPythonAnnotator
-        self.annotator = RPythonAnnotator(self, policy=policy)
+        self.annotator = RPythonAnnotator(
+            self, policy=policy, keepgoing=self.config.translation.keepgoing)
         return self.annotator
 
     def buildrtyper(self):

From pypy.commits at gmail.com  Thu Nov 24 03:12:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 00:12:02 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Tracing: constant-fold
 strhash(). Makes test_virtual_dict_constant_keys pass.
Message-ID: <5836a0d2.e626c20a.f98d5.90f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88600:19979eb85207
Date: 2016-11-24 09:11 +0100
http://bitbucket.org/pypy/pypy/changeset/19979eb85207/

Log:	Tracing: constant-fold strhash(). Makes
	test_virtual_dict_constant_keys pass. Also, in bh_strhash(), always
	compute the hash, never returns zero.

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -789,7 +790,8 @@
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
     def bh_strhash(self, s):
-        return s._obj.container.hash
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
 
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
@@ -814,7 +816,8 @@
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
     def bh_unicodehash(self, s):
-        return s._obj.container.hash
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
 
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -665,11 +666,11 @@
 
     def bh_strhash(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
-        return s.hash
+        return compute_hash(hlstr(s))
 
     def bh_unicodehash(self, string):
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
-        return u.hash
+        return compute_hash(hlunicode(u))
 
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1164,10 +1164,16 @@
 
     @arguments("box")
     def opimpl_strhash(self, strbox):
+        if isinstance(strbox, ConstPtr):
+            h = self.metainterp.cpu.bh_strhash(strbox.getref_base())
+            return ConstInt(h)
         return self.execute(rop.STRHASH, strbox)
 
     @arguments("box")
     def opimpl_unicodehash(self, unicodebox):
+        if isinstance(unicodebox, ConstPtr):
+            h = self.metainterp.cpu.bh_unicodehash(unicodebox.getref_base())
+            return ConstInt(h)
         return self.execute(rop.UNICODEHASH, unicodebox)
 
     @arguments("box")

From pypy.commits at gmail.com  Thu Nov 24 03:30:06 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 00:30:06 -0800 (PST)
Subject: [pypy-commit] pypy default: Replace 'lookup_in_type_where()[1]'
 with a method 'lookup_in_type()'.
Message-ID: <5836a50e.a285c20a.08c8.9298@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88601:5c7492207aeb
Date: 2016-11-24 09:28 +0100
http://bitbucket.org/pypy/pypy/changeset/5c7492207aeb/

Log:	Replace 'lookup_in_type_where()[1]' with a method
	'lookup_in_type()'. Implement that method directly in two fake
	object spaces.

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,8 +120,8 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
-    def lookup_in_type_where(self, cls, name):
-        return 'hopefully not needed', getattr(cls, name)
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
 
     def get_and_call_function(self, w_descr, w_obj, *args):
         return w_descr.__get__(w_obj)(*args)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -12,72 +12,52 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_str, '__getitem__')
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_unicode, '__getitem__')
 
 @specialize.memo()
 def dict_getitem(space):
     "Utility that returns the app-level descriptor dict.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_dict, '__getitem__')
 
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -339,6 +339,9 @@
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -339,6 +339,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -659,4 +663,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))

From pypy.commits at gmail.com  Thu Nov 24 03:34:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 00:34:48 -0800 (PST)
Subject: [pypy-commit] pypy default: No clue how it worked before
Message-ID: <5836a628.46bb1c0a.7da2d.5890@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88602:485b7e625505
Date: 2016-11-24 09:34 +0100
http://bitbucket.org/pypy/pypy/changeset/485b7e625505/

Log:	No clue how it worked before

diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')

From pypy.commits at gmail.com  Thu Nov 24 04:11:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 01:11:22 -0800 (PST)
Subject: [pypy-commit] pypy default: Write PyModule_GetName() based on
 PyString_AsString()
Message-ID: <5836aeba.6602c20a.c3129.96c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88603:bb6863a1903e
Date: 2016-11-24 10:01 +0100
http://bitbucket.org/pypy/pypy/changeset/bb6863a1903e/

Log:	Write PyModule_GetName() based on PyString_AsString()

diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -124,11 +124,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.bytesobject import PyString_AsString
+    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)

From pypy.commits at gmail.com  Thu Nov 24 04:27:56 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:27:56 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: raise exception on failure even
 when retval is not PyObject
Message-ID: <5836b29c.aaa3c20a.7ac19.a7b5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88605:bbee8f771d3c
Date: 2016-11-23 23:02 +0200
http://bitbucket.org/pypy/pypy/changeset/bbee8f771d3c/

Log:	raise exception on failure even when retval is not PyObject

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -875,6 +875,7 @@
         retval = fatal_value
         boxed_args = ()
         tb = None
+        state = space.fromcache(State)
         try:
             if not we_are_translated() and DEBUG_WRAPPER:
                 print >>sys.stderr, callable,
@@ -893,7 +894,6 @@
                 boxed_args += (arg_conv, )
             if pygilstate_ensure:
                 boxed_args += (args[-1], )
-            state = space.fromcache(State)
             try:
                 result = callable(space, *boxed_args)
                 if not we_are_translated() and DEBUG_WRAPPER:
@@ -939,6 +939,7 @@
         except Exception as e:
             unexpected_exception(pname, e, tb)
             _restore_gil_state(pygilstate_release, gilstate, gil_release, _gil_auto, tid)
+            state.check_and_raise_exception(always=True)
             return fatal_value
 
         assert lltype.typeOf(retval) == restype

From pypy.commits at gmail.com  Thu Nov 24 04:27:58 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:27:58 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: cleanup (arigato reviewing)
Message-ID: <5836b29e.90121c0a.32469.6be3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88606:c7ccf65c618f
Date: 2016-11-23 23:03 +0200
http://bitbucket.org/pypy/pypy/changeset/c7ccf65c618f/

Log:	cleanup (arigato reviewing)

diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -39,10 +39,11 @@
     #     we know (since we are in this function) that self is not a cpytype
     from pypy.module.cpyext.typeobject import W_PyCTypeObject
     w_type0 = w_type
-    w_mro = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
-    while w_type0.is_cpytype():
-        w_type0 = w_mro[1]
-        w_mro = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    mro_w = space.listview(space.getattr(w_type0, space.wrap('__mro__')))
+    for w_m in mro_w[1:]:
+        if not w_type0.is_cpytype():
+            break
+        w_type0 = w_m
     w_impl = space.getattr(w_type0, space.wrap('__new__'))
     args = Arguments(space, [w_type],
                      w_stararg=w_args, w_starstararg=w_kwds)

From pypy.commits at gmail.com  Thu Nov 24 04:27:54 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:27:54 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <5836b29a.61adc20a.22fed.a393@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88604:fb51b53743e3
Date: 2016-11-23 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/fb51b53743e3/

Log:	merge default into branch

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -111,7 +111,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -125,7 +127,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -355,7 +359,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -374,7 +378,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,6 +120,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type_where(self, cls, name):
+        return 'hopefully not needed', getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -805,3 +811,19 @@
             assert str(e) == "myerror"
         else:
             assert False, "Expected TypeError"
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -199,3 +210,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -332,12 +332,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -478,7 +478,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -872,14 +872,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -61,16 +61,24 @@
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_str,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
+                                                  '__getitem__')
+    return w_getitem
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
+                                                  '__getitem__')
+    return w_getitem
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
+                                                  '__getitem__')
+    return w_getitem
+
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -399,9 +399,9 @@
 
     def optimize_INT_EQ(self, op):
         arg0 = self.get_box_replacement(op.getarg(0))
+        b1 = self.getintbound(arg0)
         arg1 = self.get_box_replacement(op.getarg(1))
-        b1 = self.getintbound(op.getarg(0))
-        b2 = self.getintbound(op.getarg(1))
+        b2 = self.getintbound(arg1)
         if b1.known_gt(b2):
             self.make_constant_int(op, 0)
         elif b1.known_lt(b2):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -323,6 +323,8 @@
 def register_replacement_for(replaced_function, sandboxed_name=None):
     def wrap(func):
         from rpython.rtyper.extregistry import ExtRegistryEntry
+        # to support calling func directly
+        func._sandbox_external_name = sandboxed_name
         class ExtRegistry(ExtRegistryEntry):
             _about_ = replaced_function
             def compute_annotation(self):
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -1,6 +1,7 @@
 """Float constants"""
 
 import math, struct
+from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p, expm1
 
 from rpython.annotator.model import SomeString, SomeChar
 from rpython.rlib import objectmodel, unroll
@@ -184,104 +185,6 @@
 INFINITY = 1e200 * 1e200
 NAN = abs(INFINITY / INFINITY)    # bah, INF/INF gives us -NAN?
 
-try:
-    # Try to get math functions added in 2.6.
-    from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p
-except ImportError:
-    @not_rpython
-    def isinf(x):
-        return x == INFINITY or x == -INFINITY
-
-    @not_rpython
-    def isnan(v):
-        return v != v
-
-    @not_rpython
-    def copysign(x, y):
-        """Return x with the sign of y"""
-        if x < 0.:
-            x = -x
-        if y > 0. or (y == 0. and math.atan2(y, -1.) > 0.):
-            return x
-        else:
-            return -x
-
-    _2_to_m28 = 3.7252902984619141E-09; # 2**-28
-    _2_to_p28 = 268435456.0; # 2**28
-    _ln2 = 6.93147180559945286227E-01
-
-    @not_rpython
-    def acosh(x):
-        if isnan(x):
-            return NAN
-        if x < 1.:
-            raise ValueError("math domain error")
-        if x >= _2_to_p28:
-            if isinf(x):
-                return x
-            else:
-                return math.log(x) + _ln2
-        if x == 1.:
-            return 0.
-        if x >= 2.:
-            t = x * x
-            return math.log(2. * x - 1. / (x + math.sqrt(t - 1.0)))
-        t = x - 1.0
-        return log1p(t + math.sqrt(2. * t + t * t))
-
-    @not_rpython
-    def asinh(x):
-        absx = abs(x)
-        if not isfinite(x):
-            return x
-        if absx < _2_to_m28:
-            return x
-        if absx > _2_to_p28:
-            w = math.log(absx) + _ln2
-        elif absx > 2.:
-            w = math.log(2. * absx + 1. / (math.sqrt(x * x + 1.) + absx))
-        else:
-            t = x * x
-            w = log1p(absx + t / (1. + math.sqrt(1. + t)))
-        return copysign(w, x)
-
-    @not_rpython
-    def atanh(x):
-        if isnan(x):
-            return x
-        absx = abs(x)
-        if absx >= 1.:
-            raise ValueError("math domain error")
-        if absx < _2_to_m28:
-            return x
-        if absx < .5:
-            t = absx + absx
-            t = .5 * log1p(t + t * absx / (1. - absx))
-        else:
-            t = .5 * log1p((absx + absx) / (1. - absx))
-        return copysign(t, x)
-
-    @not_rpython
-    def log1p(x):
-        if abs(x) < DBL_EPSILON // 2.:
-            return x
-        elif -.5 <= x <= 1.:
-            y = 1. + x
-            return math.log(y) - ((y - 1.) - x) / y
-        else:
-            return math.log(1. + x)
-
-try:
-    from math import expm1 # Added in Python 2.7.
-except ImportError:
-    @not_rpython
-    def expm1(x):
-        if abs(x) < .7:
-            u = math.exp(x)
-            if u == 1.:
-                return x
-            return (u - 1.) * x / math.log(u)
-        return math.exp(x) - 1.
 
 def log2(x):
     # Uses an algorithm that should:
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -65,6 +65,24 @@
     f.close()
     assert tail == ""
 
+def test_open_dup_rposix():
+    from rpython.rlib import rposix
+    def entry_point(argv):
+        fd = rposix.open("/tmp/foobar", os.O_RDONLY, 0777)
+        assert fd == 77
+        fd2 = rposix.dup(fd)
+        assert fd2 == 78
+        return 0
+
+    exe = compile(entry_point)
+    g, f = run_in_subprocess(exe)
+    expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
+    g.close()
+    tail = f.read()
+    f.close()
+    assert tail == ""
+
 def test_read_write():
     def entry_point(argv):
         fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)

From pypy.commits at gmail.com  Thu Nov 24 04:27:59 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:27:59 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: next set of hacks,
 still not good enough to pass cython datetime tests
Message-ID: <5836b29f.0e0a1c0a.4cac2.72ac@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88607:83a4988743eb
Date: 2016-11-24 11:23 +0200
http://bitbucket.org/pypy/pypy/changeset/83a4988743eb/

Log:	next set of hacks, still not good enough to pass cython datetime
	tests

diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -21,6 +21,11 @@
     return newType;
 }
 
+void datetimetype_tp_dealloc(PyObject* self)
+{
+    return ((PyTypeObject*)typ)->tp_dealloc(self);
+}
+
 #define BASEFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_CHECKTYPES
 
 PyTypeObject footype = {
@@ -76,7 +81,7 @@
     /*tp_name*/             "foo3.datetimetype",
     /*tp_basicsize*/        sizeof(PyTypeObject),
     /*tp_itemsize*/         0,
-    /*tp_dealloc*/          0,
+    /*tp_dealloc*/          datetimetype_tp_dealloc,
     /*tp_print*/            0,
     /*tp_getattr*/          0,
     /*tp_setattr*/          0,
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1211,6 +1211,5 @@
             self._check_type_object(FooType)
         class X(object):
             __metaclass__ = FooType
-        print repr(X)
         X()
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -504,9 +504,15 @@
     this_func_ptr = llhelper(subtype_dealloc.api_func.functype,
             subtype_dealloc.api_func.get_wrapper(space))
     w_obj = from_ref(space, rffi.cast(PyObject, base))
-    # see comment in userslot.slot_tp_new, this call can also infinitely recurse if
-    # called with a c-extension type that inherits from a non-c-extension type
-    while base.c_tp_dealloc == this_func_ptr or w_obj.is_cpytype():
+    obj_not_cpytype = not w_obj.is_cpytype()
+    # see comment in userslot.slot_tp_new, this call can infinitely recurse
+    # We can only get into this function if tp_dealloc is being called on 
+    # a non-cpytype, which could or could not inherit from a cpytype
+    # So if the original obj is non-cpytype, climb the mro to the first non-cpytype,
+    # otherwise just make sure we are not calling ourselves again
+    #
+    # This logic might fail for complicated inheritance schemes.
+    while base.c_tp_dealloc == this_func_ptr or (obj_not_cpytype and w_obj.is_cpytype()):
         base = base.c_tp_base
         assert base
         w_obj = from_ref(space, rffi.cast(PyObject, base))

From pypy.commits at gmail.com  Thu Nov 24 04:28:01 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:28:01 -0800 (PST)
Subject: [pypy-commit] pypy default: add VERSION to distutils' sysconfig
Message-ID: <5836b2a1.a351c20a.ac0fb.2fcf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r88608:894c49fbaddf
Date: 2016-11-24 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/894c49fbaddf/

Log:	add VERSION to distutils' sysconfig

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -67,6 +67,7 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
     g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g

From pypy.commits at gmail.com  Thu Nov 24 04:32:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 01:32:46 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Next fix,
 now the test passes with less operations than before
Message-ID: <5836b3be.c4811c0a.2dd69.7217@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88610:e15e2e2c8a63
Date: 2016-11-24 10:32 +0100
http://bitbucket.org/pypy/pypy/changeset/e15e2e2c8a63/

Log:	Next fix, now the test passes with less operations than before

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,5 +1,5 @@
 
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
@@ -73,6 +73,9 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
+    def getstrhash(self, op, mode):
+        return None
+
     def copy_fields_to_const(self, constinfo, optheap):
         pass
 
@@ -790,6 +793,20 @@
                 return None
             return ConstInt(len(s))
 
+    def getstrhash(self, op, mode):
+        from rpython.jit.metainterp.optimizeopt import vstring
+
+        if mode is vstring.mode_string:
+            s = self._unpack_str(vstring.mode_string)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+        else:
+            s = self._unpack_str(vstring.mode_unicode)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -537,6 +537,20 @@
                 return
         return self.emit(op)
 
+    def optimize_STRHASH(self, op):
+        return self._optimize_STRHASH(op, mode_string)
+    def optimize_UNICODEHASH(self, op):
+        return self._optimize_STRHASH(op, mode_unicode)
+
+    def _optimize_STRHASH(self, op, mode):
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            lgtop = opinfo.getstrhash(op, mode)
+            if lgtop is not None:
+                self.make_equal_to(op, lgtop)
+                return
+        return self.emit(op)
+
     def optimize_COPYSTRCONTENT(self, op):
         return self._optimize_COPYSTRCONTENT(op, mode_string)
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3775,9 +3775,8 @@
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
-        self.check_resops({'int_gt': 2, 'getfield_gc_i': 1, 'int_eq': 1,
-                           'guard_true': 2, 'int_sub': 2, 'jump': 1,
-                           'guard_false': 1})
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
 
     def test_virtual_after_bridge(self):
         myjitdriver = JitDriver(greens = [], reds = ["n"])

From pypy.commits at gmail.com  Thu Nov 24 04:57:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 01:57:08 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: test and fix
Message-ID: <5836b974.88711c0a.25ac.7f45@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88611:6b96de7cd8ab
Date: 2016-11-24 10:56 +0100
http://bitbucket.org/pypy/pypy/changeset/6b96de7cd8ab/

Log:	test and fix

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1206,14 +1206,14 @@
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
-    @arguments("cpu", "i", "i", "I", "R", "d")
+    @arguments("cpu", "i", "i", "I", "R", "d", returns="i")
     def bhimpl_conditional_call_value_ir_i(cpu, value, func, args_i, args_r,
                                            calldescr):
         if value == 0:
             value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
         return value
 
-    @arguments("cpu", "r", "i", "I", "R", "d")
+    @arguments("cpu", "r", "i", "I", "R", "d", returns="r")
     def bhimpl_conditional_call_value_ir_r(cpu, value, func, args_i, args_r,
                                            calldescr):
         if not value:
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -249,13 +249,33 @@
                 p = jit.conditional_call_elidable(p, externfn, m)
                 n -= p
             return n
-        assert f(219, 5, 0) == -1
-        res = self.meta_interp(f, [219, 5, 0])
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
         self.check_resops(call_pure_i=0,
                           cond_call_value_i=2,   # ideally 1, but see above
                           int_sub=2)
 
+    def test_cond_call_in_blackhole(self):
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                if n > 6:    # will fail and finish in the blackhole
+                    pass
+                if jit.we_are_jitted():   # manually inline here
+                    p = jit._jit_conditional_call_value(p, externfn, m)
+                else:
+                    p = jit.conditional_call_elidable(p, externfn, m)
+                n -= p
+            return n
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+
 
 class TestCall(LLJitMixin, CallTest):
     pass

From pypy.commits at gmail.com  Thu Nov 24 05:00:12 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:12 -0800 (PST)
Subject: [pypy-commit] stmgc default: silence flycheck with a fake #include
Message-ID: <5836ba2c.52301c0a.d82bc.8388@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: 
Changeset: r1996:b0d76f59efe5
Date: 2016-11-15 17:18 +0100
http://bitbucket.org/pypy/stmgc/changeset/b0d76f59efe5/

Log:	silence flycheck with a fake #include

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 char *stm_object_pages;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -1,3 +1,9 @@
+#ifndef _STMGC_H
+# error "must be compiled via stmgc.c"
+# include "../stmgc.h"  // silence flymake
+#endif
+
+
 #define _STM_CORE_H_
 
 #include <stdlib.h>
@@ -8,7 +14,6 @@
 #include <pthread.h>
 #include <signal.h>
 
-
 /************************************************************/
 
 #ifndef STM_GC_NURSERY
diff --git a/c8/stm/detach.c b/c8/stm/detach.c
--- a/c8/stm/detach.c
+++ b/c8/stm/detach.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 #include <errno.h>
diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -1,4 +1,7 @@
-
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
+#endif
 
 /* callbacks */
 void (*stmcb_light_finalizer)(object_t *);
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 static struct tree_s *tree_prebuilt_objs = NULL;     /* XXX refactor */
diff --git a/c8/stm/hash_id.c b/c8/stm/hash_id.c
--- a/c8/stm/hash_id.c
+++ b/c8/stm/hash_id.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
--- a/c8/stm/hashtable.c
+++ b/c8/stm/hashtable.c
@@ -40,7 +40,7 @@
 
 Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
 */
-
+#include <stdint.h>
 
 uint32_t stm_hashtable_entry_userdata;
 
diff --git a/c8/stm/largemalloc.c b/c8/stm/largemalloc.c
--- a/c8/stm/largemalloc.c
+++ b/c8/stm/largemalloc.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 /* This contains a lot of inspiration from malloc() in the GNU C Library.
diff --git a/c8/stm/list.c b/c8/stm/list.c
--- a/c8/stm/list.c
+++ b/c8/stm/list.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/marker.c b/c8/stm/marker.c
--- a/c8/stm/marker.c
+++ b/c8/stm/marker.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 /************************************************************/
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 #include <unistd.h>
diff --git a/c8/stm/prebuilt.c b/c8/stm/prebuilt.c
--- a/c8/stm/prebuilt.c
+++ b/c8/stm/prebuilt.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // for auto-completion
 #endif
 
 #include <signal.h>
diff --git a/c8/stm/signal_handler.c b/c8/stm/signal_handler.c
--- a/c8/stm/signal_handler.c
+++ b/c8/stm/signal_handler.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -5,6 +5,7 @@
 
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 
diff --git a/c8/stm/weakref.c b/c8/stm/weakref.c
--- a/c8/stm/weakref.c
+++ b/c8/stm/weakref.c
@@ -1,5 +1,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
+# include "core.h"  // silence flymake
 #endif
 
 #define WEAKREF_PTR(wr, sz)  ((object_t * TLPREFIX *)(((stm_char *)(wr)) + (sz) - sizeof(void*)))

From pypy.commits at gmail.com  Thu Nov 24 05:00:14 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:14 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: rename light_finalizer to
 destructor
Message-ID: <5836ba2e.8d071c0a.3f205.79b3@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r1997:2101463c93d1
Date: 2016-11-16 08:59 +0100
http://bitbucket.org/pypy/stmgc/changeset/2101463c93d1/

Log:	rename light_finalizer to destructor

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1047,7 +1047,7 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
-    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_destructors));
     assert(STM_PSEGMENT->finalizers == NULL);
     assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -13,6 +13,7 @@
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdbool.h>
 
 /************************************************************/
 
@@ -145,8 +146,8 @@
 #endif
 
     /* light finalizers */
-    struct list_s *young_objects_with_light_finalizers;
-    struct list_s *old_objects_with_light_finalizers;
+    struct list_s *young_objects_with_destructors;
+    struct list_s *old_objects_with_destructors;
 
     /* regular finalizers (objs from the current transaction only) */
     struct finalizers_s *finalizers;
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -4,7 +4,7 @@
 #endif
 
 /* callbacks */
-void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_destructor)(object_t *);
 void (*stmcb_finalizer)(object_t *);
 
 
@@ -91,7 +91,7 @@
     char *old_gs_register = STM_SEGMENT->segment_base;
     bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
 
-    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    struct list_s *lst = pseg->young_objects_with_destructors;
     long i, count = list_count(lst);
     if (lst > 0) {
         for (i = 0; i < count; i++) {
@@ -101,15 +101,15 @@
                 set_gs_register(pseg->pub.segment_base);
                 must_fix_gs = false;
             }
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         list_clear(lst);
     }
 
     /* also deals with overflow objects: they are at the tail of
-       old_objects_with_light_finalizers (this list is kept in order
+       old_objects_with_destructors (this list is kept in order
        and we cannot add any already-committed object) */
-    lst = pseg->old_objects_with_light_finalizers;
+    lst = pseg->old_objects_with_destructors;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
@@ -120,7 +120,7 @@
             set_gs_register(pseg->pub.segment_base);
             must_fix_gs = false;
         }
-        stmcb_light_finalizer(obj);
+        stmcb_destructor(obj);
     }
 
     if (STM_SEGMENT->segment_base != old_gs_register)
@@ -128,14 +128,14 @@
 }
 
 
-void stm_enable_light_finalizer(object_t *obj)
+void stm_enable_destructor(object_t *obj)
 {
     if (_is_young(obj)) {
-        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_destructors, obj);
     }
     else {
         assert(_is_from_same_transaction(obj));
-        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
     }
 }
 
@@ -165,7 +165,7 @@
 {
     /* for light finalizers: executes finalizers for objs that don't survive
        this minor gc */
-    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_destructors;
     long i, count = list_count(lst);
     for (i = 0; i < count; i++) {
         object_t *obj = (object_t *)list_item(lst, i);
@@ -174,12 +174,12 @@
         object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
         if (pforwarded_array[0] != GCWORD_MOVED) {
             /* not moved: the object dies */
-            stmcb_light_finalizer(obj);
+            stmcb_destructor(obj);
         }
         else {
             obj = pforwarded_array[1]; /* moved location */
             assert(!_is_young(obj));
-            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_destructors, obj);
         }
     }
     list_clear(lst);
@@ -191,11 +191,11 @@
     int old_gs_register = STM_SEGMENT->segment_num;
     int current_gs_register = old_gs_register;
     long j;
-    assert(list_is_empty(get_priv_segment(0)->old_objects_with_light_finalizers));
+    assert(list_is_empty(get_priv_segment(0)->old_objects_with_destructors));
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
 
-        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        struct list_s *lst = pseg->old_objects_with_destructors;
         long i, count = list_count(lst);
         lst->count = 0;
         for (i = 0; i < count; i++) {
@@ -217,7 +217,7 @@
                     set_gs_register(get_segment_base(j));
                     current_gs_register = j;
                 }
-                stmcb_light_finalizer(obj);
+                stmcb_destructor(obj);
             }
             else {
                 /* object survives */
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -1,3 +1,4 @@
+#include <stdint.h>
 
 /* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -100,8 +100,8 @@
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
         pr->callbacks_on_commit_and_abort[1] = tree_create();
-        pr->young_objects_with_light_finalizers = list_create();
-        pr->old_objects_with_light_finalizers = list_create();
+        pr->young_objects_with_destructors = list_create();
+        pr->old_objects_with_destructors = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
@@ -156,8 +156,8 @@
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
         tree_free(pr->callbacks_on_commit_and_abort[1]);
-        list_free(pr->young_objects_with_light_finalizers);
-        list_free(pr->old_objects_with_light_finalizers);
+        list_free(pr->young_objects_with_destructors);
+        list_free(pr->old_objects_with_destructors);
         if (pr->active_queues) tree_free(pr->active_queues);
     }
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -710,11 +710,11 @@
 void stm_force_transaction_break(stm_thread_local_t *tl);
 
 
-/* Support for light finalizers.  This is a simple version of
+/* Support for destructors.  This is a simple version of
    finalizers that guarantees not to do anything fancy, like not
    resurrecting objects. */
-extern void (*stmcb_light_finalizer)(object_t *);
-void stm_enable_light_finalizer(object_t *);
+extern void (*stmcb_destructor)(object_t *);
+void stm_enable_destructor(object_t *);
 
 /* Support for regular finalizers.  Unreachable objects with
    finalizers are kept alive, as well as everything they point to, and
@@ -742,7 +742,7 @@
    'object_t *'.  Note that the type 'stm_hashtable_t' is not an
    object type at all; you need to allocate and free it explicitly.
    If you want to embed the hashtable inside an 'object_t' you
-   probably need a light finalizer to do the freeing. */
+   probably need a destructor to do the freeing. */
 typedef struct stm_hashtable_s stm_hashtable_t;
 typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
 
@@ -797,8 +797,8 @@
 /* Queues.  The items you put() and get() back are in random order.
    Like hashtables, the type 'stm_queue_t' is not an object type at
    all; you need to allocate and free it explicitly.  If you want to
-   embed the queue inside an 'object_t' you probably need a light
-   finalizer to do the freeing. */
+   embed the queue inside an 'object_t' you probably need a destructor
+   to do the freeing. */
 typedef struct stm_queue_s stm_queue_t;
 
 stm_queue_t *stm_queue_create(void);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -206,9 +206,8 @@
 bool (*_stm_smallmalloc_keep)(char *data);
 void _stm_smallmalloc_sweep_test(void);
 
-
-void (*stmcb_light_finalizer)(object_t *);
-void stm_enable_light_finalizer(object_t *);
+void (*stmcb_destructor)(object_t *);
+void stm_enable_destructor(object_t *);
 
 void (*stmcb_finalizer)(object_t *);
 
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -2,13 +2,13 @@
 import py
 
 
-class TestLightFinalizer(BaseTest):
+class TestDestructors(BaseTest):
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
+        def destructor(obj):
             assert stm_get_obj_size(obj) == 48
             segnum = lib.current_segment_num()
             tlnum = '?'
@@ -17,21 +17,21 @@
                     if tl.last_associated_segment_num == segnum:
                         tlnum = n
                         break
-            self.light_finalizers_called.append((obj, tlnum))
-        self.light_finalizers_called = []
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+            self.destructors_called.append((obj, tlnum))
+        self.destructors_called = []
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
 
     def teardown_method(self, meth):
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         BaseTest.teardown_method(self, meth)
 
     def expect_finalized(self, objs, from_tlnum=None):
-        assert [obj for (obj, tlnum) in self.light_finalizers_called] == objs
+        assert [obj for (obj, tlnum) in self.destructors_called] == objs
         if from_tlnum is not None:
-            for obj, tlnum in self.light_finalizers_called:
+            for obj, tlnum in self.destructors_called:
                 assert tlnum == from_tlnum
-        self.light_finalizers_called = []
+        self.destructors_called = []
 
     def test_no_finalizer(self):
         self.start_transaction()
@@ -39,45 +39,45 @@
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_young_light_finalizer(self):
+    def test_young_destructor(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.expect_finalized([])
         self.commit_transaction()
         self.expect_finalized([lp1], from_tlnum=0)
 
-    def test_young_light_finalizer_survives(self):
+    def test_young_destructor_survives(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)       # stays alive
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_young_light_finalizer_aborts(self):
+    def test_young_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.expect_finalized([])
         self.abort_transaction()
         self.start_transaction()
         self.expect_finalized([lp1], from_tlnum=0)
 
-    def test_old_light_finalizer(self):
+    def test_old_destructor(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_old_light_finalizer_2(self):
+    def test_old_destructor_2(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -86,10 +86,10 @@
         self.expect_finalized([lp1])
         self.commit_transaction()
 
-    def test_old_light_finalizer_survives(self):
+    def test_old_destructor_survives(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -98,13 +98,13 @@
         self.commit_transaction()
         self.expect_finalized([])
 
-    def test_old_light_finalizer_segment(self):
+    def test_old_destructor_segment(self):
         self.start_transaction()
         #
         self.switch(1)
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
@@ -114,10 +114,10 @@
         stm_major_collect()
         self.expect_finalized([lp1], from_tlnum=1)
 
-    def test_old_light_finalizer_aborts(self):
+    def test_old_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         self.commit_transaction()
         #
@@ -126,10 +126,10 @@
         self.abort_transaction()
         self.expect_finalized([])
 
-    def test_overflow_light_finalizer_aborts(self):
+    def test_overflow_destructor_aborts(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
-        lib.stm_enable_light_finalizer(lp1)
+        lib.stm_enable_destructor(lp1)
         self.push_root(lp1)
         stm_minor_collect()
         lp1 = self.pop_root()
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
--- a/c8/test/test_hashtable.py
+++ b/c8/test/test_hashtable.py
@@ -43,8 +43,8 @@
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
-            print 'light_finalizer:', obj
+        def destructor(obj):
+            print 'destructor:', obj
             try:
                 assert lib._get_type_id(obj) == 421419
                 self.seen_hashtables -= 1
@@ -54,20 +54,20 @@
                 self.errors.append(sys.exc_info()[2])
                 raise
 
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
         self.seen_hashtables = 0
         self.errors = []
 
     def teardown_method(self, meth):
         BaseTest.teardown_method(self, meth)
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         assert self.errors == []
         assert self.seen_hashtables == 0
 
     def allocate_hashtable(self):
         h = stm_allocate_hashtable()
-        lib.stm_enable_light_finalizer(h)
+        lib.stm_enable_destructor(h)
         self.seen_hashtables += 1
         return h
 
diff --git a/c8/test/test_queue.py b/c8/test/test_queue.py
--- a/c8/test/test_queue.py
+++ b/c8/test/test_queue.py
@@ -13,8 +13,8 @@
         BaseTest.setup_method(self, meth)
         #
         @ffi.callback("void(object_t *)")
-        def light_finalizer(obj):
-            print 'light_finalizer:', obj
+        def destructor(obj):
+            print 'destructor:', obj
             try:
                 assert lib._get_type_id(obj) == 421417
                 self.seen_queues -= 1
@@ -24,20 +24,20 @@
                 self.errors.append(sys.exc_info()[2])
                 raise
 
-        lib.stmcb_light_finalizer = light_finalizer
-        self._light_finalizer_keepalive = light_finalizer
+        lib.stmcb_destructor = destructor
+        self._destructor_keepalive = destructor
         self.seen_queues = 0
         self.errors = []
 
     def teardown_method(self, meth):
         BaseTest.teardown_method(self, meth)
-        lib.stmcb_light_finalizer = ffi.NULL
+        lib.stmcb_destructor = ffi.NULL
         assert self.errors == []
         assert self.seen_queues == 0
 
     def allocate_queue(self):
         q = stm_allocate_queue()
-        lib.stm_enable_light_finalizer(q)
+        lib.stm_enable_destructor(q)
         self.seen_queues += 1
         return q
 

From pypy.commits at gmail.com  Thu Nov 24 05:00:16 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:16 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: silence a few more flycheck
 errors
Message-ID: <5836ba30.aaa3c20a.7ac19.b4c1@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r1998:75b748b315da
Date: 2016-11-16 09:17 +0100
http://bitbucket.org/pypy/stmgc/changeset/75b748b315da/

Log:	silence a few more flycheck errors

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -14,6 +14,7 @@
 #include <pthread.h>
 #include <signal.h>
 #include <stdbool.h>
+#include "list.h"
 
 /************************************************************/
 
diff --git a/c8/stm/fprintcolor.h b/c8/stm/fprintcolor.h
--- a/c8/stm/fprintcolor.h
+++ b/c8/stm/fprintcolor.h
@@ -1,3 +1,7 @@
+#ifndef _FPRINTCOLOR_H
+#define _FPRINTCOLOR_H
+
+
 /* ------------------------------------------------------------ */
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
@@ -40,3 +44,5 @@
 __attribute__((unused))
 static void stm_fatalerror(const char *format, ...)
      __attribute__((format (printf, 1, 2), noreturn));
+
+#endif
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -1,5 +1,11 @@
+#ifndef _LIST_H
+#define _LIST_H
+
+
 #include <stdlib.h>
 #include <stdbool.h>
+#include <stdint.h>
+
 
 /************************************************************/
 
@@ -245,3 +251,5 @@
     TREE_FIND(tree, addr, result, return false);
     return true;
 }
+
+#endif
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -5,6 +5,8 @@
 
 #include <signal.h>
 #include <fcntl.h>           /* For O_* constants */
+#include "signal_handler.h"
+#include "fprintcolor.h"
 
 static void setup_mmap(char *reason)
 {

From pypy.commits at gmail.com  Thu Nov 24 04:28:03 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 01:28:03 -0800 (PST)
Subject: [pypy-commit] pypy missing-tp_new: merge default into branch
Message-ID: <5836b2a3.c5371c0a.a9d9e.6ce3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: missing-tp_new
Changeset: r88609:adafff615169
Date: 2016-11-24 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/adafff615169/

Log:	merge default into branch

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -67,6 +67,7 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
     g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,8 +120,8 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
-    def lookup_in_type_where(self, cls, name):
-        return 'hopefully not needed', getattr(cls, name)
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
 
     def get_and_call_function(self, w_descr, w_obj, *args):
         return w_descr.__get__(w_obj)(*args)
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -124,11 +124,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.bytesobject import PyString_AsString
+    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -12,72 +12,52 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_str, '__getitem__')
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_unicode, '__getitem__')
 
 @specialize.memo()
 def dict_getitem(space):
     "Utility that returns the app-level descriptor dict.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_dict, '__getitem__')
 
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -339,6 +339,9 @@
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -339,6 +339,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -659,4 +663,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -22,7 +22,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -50,6 +51,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -202,6 +206,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -348,6 +358,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -392,6 +404,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -482,6 +498,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -67,7 +67,8 @@
         if self.annotator is not None:
             raise ValueError("we already have an annotator")
         from rpython.annotator.annrpython import RPythonAnnotator
-        self.annotator = RPythonAnnotator(self, policy=policy)
+        self.annotator = RPythonAnnotator(
+            self, policy=policy, keepgoing=self.config.translation.keepgoing)
         return self.annotator
 
     def buildrtyper(self):

From pypy.commits at gmail.com  Thu Nov 24 05:00:23 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:23 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: re-introduce running
 finalizers within a regular transaction
Message-ID: <5836ba37.849c1c0a.38a6c.7dd0@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r2000:ceda93abb51f
Date: 2016-11-17 10:39 +0100
http://bitbucket.org/pypy/stmgc/changeset/ceda93abb51f/

Log:	re-introduce running finalizers within a regular transaction

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1206,6 +1206,8 @@
 
 static void _core_commit_transaction(bool external)
 {
+    exec_local_finalizers();
+
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->transaction_state != TS_NONE);
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -18,6 +18,7 @@
     f->probably_young_objects_with_finalizers = list_create();
     f->run_finalizers = list_create();
     f->running_next = NULL;
+    f->lock = 0;
 }
 
 static void setup_finalizer(void)
@@ -493,6 +494,68 @@
         getfield on *dying obj*).
 */
 
+static bool _trigger_finalizer_queues(struct finalizers_s *f)
+{
+    /* runs triggers of finalizer queues that have elements in the queue. May
+       run outside of a transaction, as triggers should be safe to call(?)
+
+       returns true if there are old-style finalizers to run */
+
+    bool trigger_oldstyle = false;
+    bool *to_trigger = calloc(g_finalizer_triggers.count, sizeof(bool));
+
+    while (__sync_lock_test_and_set(&f->lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        stm_spin_loop();
+    }
+
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        dprintf(("qindex=%d\n", qindex));
+        if (qindex == -1)
+            trigger_oldstyle = true;
+        else
+            to_trigger[qindex] = true;
+    }
+
+    __sync_lock_release(&f->lock);
+
+    // trigger now:
+    for (int i = 0; i < g_finalizer_triggers.count; i++) {
+        if (to_trigger[i]) {
+            dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
+            // XXX: check that triggers *really* cannot touch GC-memory,
+            // otherwise, this needs to run in an (inevitable) transaction
+#ifndef NDEBUG
+            set_gs_register(NULL);
+#endif
+            g_finalizer_triggers.triggers[i]();
+        }
+    }
+
+    free(to_trigger);
+
+    return trigger_oldstyle;
+}
+
+static void _invoke_local_finalizers()
+{
+    /* called inside a transaction; invoke local triggers, process old-style
+     * local finalizers */
+    dprintf(("invoke_local_finalizers %lu\n", list_count(STM_PSEGMENT->finalizers->run_finalizers)));
+    if (list_is_empty(STM_PSEGMENT->finalizers->run_finalizers))
+        return;
+
+    if (!_trigger_finalizer_queues(STM_PSEGMENT->finalizers))
+        return; // no oldstyle to run
+
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        stmcb_finalizer(obj);
+    }
+}
+
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
@@ -503,38 +566,8 @@
     if (list_is_empty(g_finalizers.run_finalizers))
         return;
 
-    bool *to_trigger = calloc(g_finalizer_triggers.count + 1, sizeof(bool));
-
-    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
-        /* somebody is adding more finalizers (_commit_finalizer()) */
-        stm_spin_loop();
-    }
-
-    int count = list_count(g_finalizers.run_finalizers);
-    for (int i = 0; i < count; i += 2) {
-        int qindex = (int)list_item(g_finalizers.run_finalizers, i + 1);
-        to_trigger[qindex + 1] = true; // also for -1!
-    }
-
-    __sync_lock_release(&g_finalizers.lock);
-
-    // trigger now:
-    for (int i = 1; i < g_finalizer_triggers.count; i++) {
-        if (to_trigger[i]) {
-            dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
-            // XXX: check that triggers *really* cannot touch GC-memory,
-            // otherwise, this needs to run in an (inevitable) transaction
-            g_finalizer_triggers.triggers[i - 1]();
-        }
-    }
-
-    if (!to_trigger[0]) {
-        // 0 is the qindex for old-style finalizers,
-        // so nothing todo here.
-        free(to_trigger);
-        return;
-    }
-    free(to_trigger);
+    if (!_trigger_finalizer_queues(&g_finalizers))
+        return; // no oldstyle finalizers to run
 
     // run old-style finalizers:
     dprintf(("invoke-oldstyle-finalizers\n"));
@@ -553,6 +586,29 @@
 }
 
 object_t* stm_next_to_finalize(int queue_index) {
+    /* first check local run_finalizers queue, then global */
+    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
+        struct list_s *lst = STM_PSEGMENT->finalizers->run_finalizers;
+        int count = list_count(lst);
+        for (int i = 0; i < count; i += 2) {
+            int qindex = (int)list_item(lst, i + 1);
+            if (qindex == queue_index) {
+                /* no need to become inevitable for local ones */
+                /* Remove obj from list and return it. */
+                object_t *obj = (object_t*)list_item(lst, i);
+                if (i < count - 2) {
+                    memmove(&lst->items[i],
+                            &lst->items[i + 2],
+                            (count - 2) * sizeof(uintptr_t));
+                }
+                lst->count -= 2;
+                return obj;
+            }
+        }
+    }
+
+    /* no local finalizers found, continue in global list */
+
     while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
         /* somebody is adding more finalizers (_commit_finalizer()) */
         stm_spin_loop();
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -38,9 +38,16 @@
 
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl);
+static void _invoke_local_finalizers(void);
 
 #define invoke_general_finalizers(tl)    do {   \
      _invoke_general_finalizers(tl);         \
 } while (0)
 
+
+#define exec_local_finalizers()  do {                   \
+    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) \
+        _invoke_local_finalizers();  \
+} while (0)
+
 #endif
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -213,6 +213,7 @@
     }
 
     s_mutex_unlock();
+    exec_local_finalizers();
 }
 
 
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -311,6 +311,7 @@
     assert(_stm_in_transaction(tl));
     ensure_gs_register(tl->last_associated_segment_num);
     assert(STM_SEGMENT->running_thread == tl);
+    exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -179,6 +179,8 @@
         if isinstance(objs, int):
             assert len(self.finalizers_called) == objs
         else:
+            print objs
+            print self.finalizers_called
             assert self.finalizers_called == objs
         self.finalizers_called = []
 
@@ -186,14 +188,12 @@
         self.start_transaction()
         lp1 = stm_allocate(48)
         stm_major_collect()
-        self.commit_transaction()
         self.expect_finalized([])
 
     def test_no_finalizer_in_minor_collection(self):
         self.start_transaction()
         lp1 = stm_allocate_with_finalizer(48)
         stm_minor_collect()
-        self.commit_transaction()
         self.expect_finalized([])
 
     def test_finalizer_in_major_collection(self):
@@ -204,21 +204,20 @@
             lp3 = stm_allocate_with_finalizer(48)
             self.expect_finalized([])
             self.push_roots([lp1, lp2, lp3])
-            self.commit_transaction()  # move finalizer-objs to global queue
-            self.start_transaction()
+            stm_minor_collect()
             lp1, lp2, lp3 = self.pop_roots()
             print repeat, lp1, lp2, lp3
             stm_major_collect()
-            self.commit_transaction()  # invoke finalizers
             self.expect_finalized([lp1, lp2, lp3])
-            self.start_transaction()
-        self.commit_transaction()
 
     def test_finalizer_from_other_thread(self):
         self.start_transaction()
         lp1 = stm_allocate_with_finalizer(48)
         stm_set_char(lp1, 'H')
         self.expect_content_character = 'H'
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
         print lp1
         #
         self.switch(1)
@@ -227,7 +226,6 @@
         self.expect_finalized([])      # marked as dead, but wrong thread
         #
         self.switch(0)
-        py.test.xfail("we don't finalize in the same transaction anymore.")
         self.expect_finalized([lp1])   # now it has been finalized
 
     def test_finalizer_ordering(self):
@@ -240,12 +238,10 @@
         stm_set_ref(lp1, 0, lp2)
 
         self.push_roots([lp1, lp2, lp3])
-        self.commit_transaction()  # move finalizer-objs to global queue
-        self.start_transaction()
+        stm_minor_collect()
         lp1, lp2, lp3 = self.pop_roots()
 
         stm_major_collect()
-        self.commit_transaction() # invoke finalizers
         self.expect_finalized([lp3])
 
     def test_finalizer_extra_transaction(self):
@@ -280,11 +276,8 @@
 
         self.expect_finalized([])
         stm_major_collect()
-        self.commit_transaction()
-        self.expect_finalized(1)
         self.switch(0)
-        self.commit_transaction()
-        self.expect_finalized(1)
+        self.expect_finalized(2)
 
     def test_run_major_collect_in_finalizer(self):
         self.run_major_collect_in_finalizer = True
@@ -294,7 +287,6 @@
         lp3 = stm_allocate_with_finalizer(32)
         print lp1, lp2, lp3
         stm_major_collect()
-        self.commit_transaction()
 
     def test_new_objects_w_finalizers(self):
         self.switch(2)
@@ -311,5 +303,4 @@
 
         self.expect_finalized([])
         stm_major_collect()
-        self.commit_transaction()
         self.expect_finalized([lp1])

From pypy.commits at gmail.com  Thu Nov 24 05:00:17 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:17 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: WIP: new finalizer API
Message-ID: <5836ba31.e576c20a.d6e2c.ab73@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r1999:a59a745d2093
Date: 2016-11-16 16:48 +0100
http://bitbucket.org/pypy/stmgc/changeset/a59a745d2093/

Log:	WIP: new finalizer API

	oldstyle-finalizers should still work, but the new finalizer queues
	are untested as of this commit.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1048,7 +1048,6 @@
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
     assert(list_is_empty(STM_PSEGMENT->young_objects_with_destructors));
-    assert(STM_PSEGMENT->finalizers == NULL);
     assert(STM_PSEGMENT->active_queues == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
@@ -1207,8 +1206,6 @@
 
 static void _core_commit_transaction(bool external)
 {
-    exec_local_finalizers();
-
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->transaction_state != TS_NONE);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -146,7 +146,7 @@
     pthread_t running_pthread;
 #endif
 
-    /* light finalizers */
+    /* destructors */
     struct list_s *young_objects_with_destructors;
     struct list_s *old_objects_with_destructors;
 
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -2,6 +2,10 @@
 # error "must be compiled via stmgc.c"
 # include "core.h"  // silence flymake
 #endif
+#include "finalizer.h"
+#include "fprintcolor.h"
+#include "nursery.h"
+#include "gcpage.h"
 
 /* callbacks */
 void (*stmcb_destructor)(object_t *);
@@ -11,22 +15,51 @@
 static void init_finalizers(struct finalizers_s *f)
 {
     f->objects_with_finalizers = list_create();
-    f->count_non_young = 0;
-    f->run_finalizers = NULL;
+    f->probably_young_objects_with_finalizers = list_create();
+    f->run_finalizers = list_create();
     f->running_next = NULL;
 }
 
 static void setup_finalizer(void)
 {
     init_finalizers(&g_finalizers);
+
+    for (long j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        assert(pseg->finalizers == NULL);
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        pseg->finalizers = f;
+    }
 }
 
-static void teardown_finalizer(void)
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers)
 {
-    if (g_finalizers.run_finalizers != NULL)
-        list_free(g_finalizers.run_finalizers);
-    list_free(g_finalizers.objects_with_finalizers);
+    assert(g_finalizer_triggers.count == 0);
+    assert(g_finalizer_triggers.triggers == NULL);
+
+    g_finalizer_triggers.count = number;
+    g_finalizer_triggers.triggers = (stm_finalizer_trigger_fn *)
+        malloc(number * sizeof(stm_finalizer_trigger_fn));
+
+    for (int qindex = 0; qindex < number; qindex++) {
+        g_finalizer_triggers.triggers[qindex] = triggers[qindex];
+        dprintf(("setup_finalizer_queue(qindex=%d,fun=%p)\n", qindex, triggers[qindex]));
+    }
+}
+
+static void teardown_finalizer(void) {
+    LIST_FREE(g_finalizers.run_finalizers);
+    LIST_FREE(g_finalizers.objects_with_finalizers);
+    LIST_FREE(g_finalizers.probably_young_objects_with_finalizers);
     memset(&g_finalizers, 0, sizeof(g_finalizers));
+
+    if (g_finalizer_triggers.triggers)
+        free(g_finalizer_triggers.triggers);
+    memset(&g_finalizer_triggers, 0, sizeof(g_finalizer_triggers));
 }
 
 static void _commit_finalizers(void)
@@ -36,7 +69,7 @@
         stm_spin_loop();
     }
 
-    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
            'g_finalizers.run_finalizers', dropping any initial NULLs
            (finalizers already called) */
@@ -48,24 +81,24 @@
             *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
         }
         if (frm < list_count(src)) {
-            if (g_finalizers.run_finalizers == NULL)
-                g_finalizers.run_finalizers = list_create();
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
                 src, frm);
         }
-        list_free(src);
     }
+    LIST_FREE(STM_PSEGMENT->finalizers->run_finalizers);
 
     /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
        into 'g_finalizers.objects_with_finalizers' */
     g_finalizers.objects_with_finalizers = list_extend(
         g_finalizers.objects_with_finalizers,
         STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
-    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+    assert(list_is_empty(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers));
+    LIST_FREE(STM_PSEGMENT->finalizers->objects_with_finalizers);
+    LIST_FREE(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers);
 
-    free(STM_PSEGMENT->finalizers);
-    STM_PSEGMENT->finalizers = NULL;
+    // re-init
+    init_finalizers(STM_PSEGMENT->finalizers);
 
     __sync_lock_release(&g_finalizers.lock);
 }
@@ -74,17 +107,14 @@
 {
     /* like _commit_finalizers(), but forget everything from the
        current transaction */
-    if (pseg->finalizers != NULL) {
-        if (pseg->finalizers->run_finalizers != NULL) {
-            if (pseg->finalizers->running_next != NULL) {
-                *pseg->finalizers->running_next = (uintptr_t)-1;
-            }
-            list_free(pseg->finalizers->run_finalizers);
-        }
-        list_free(pseg->finalizers->objects_with_finalizers);
-        free(pseg->finalizers);
-        pseg->finalizers = NULL;
+    if (pseg->finalizers->running_next != NULL) {
+        *pseg->finalizers->running_next = (uintptr_t)-1;
     }
+    LIST_FREE(pseg->finalizers->run_finalizers);
+    LIST_FREE(pseg->finalizers->objects_with_finalizers);
+    LIST_FREE(pseg->finalizers->probably_young_objects_with_finalizers);
+    // re-init
+    init_finalizers(pseg->finalizers);
 
     /* call the light finalizers for objects that are about to
        be forgotten from the current transaction */
@@ -139,31 +169,29 @@
     }
 }
 
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+
+void stm_enable_finalizer(int queue_index, object_t *obj)
 {
-    object_t *obj = _stm_allocate_external(size_rounded_up);
-
-    if (STM_PSEGMENT->finalizers == NULL) {
-        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
-        if (f == NULL)
-            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
-        init_finalizers(f);
-        STM_PSEGMENT->finalizers = f;
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers, queue_index);
     }
-    assert(STM_PSEGMENT->finalizers->count_non_young
-           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
-    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
-    return obj;
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, queue_index);
+    }
 }
 
 
+
 /************************************************************/
-/*  Light finalizers
+/*  Destructors
 */
 
-static void deal_with_young_objects_with_finalizers(void)
+static void deal_with_young_objects_with_destructors(void)
 {
-    /* for light finalizers: executes finalizers for objs that don't survive
+    /* for destructors: executes destructors for objs that don't survive
        this minor gc */
     struct list_s *lst = STM_PSEGMENT->young_objects_with_destructors;
     long i, count = list_count(lst);
@@ -185,9 +213,9 @@
     list_clear(lst);
 }
 
-static void deal_with_old_objects_with_finalizers(void)
+static void deal_with_old_objects_with_destructors(void)
 {
-    /* for light finalizers */
+    /* for destructors */
     int old_gs_register = STM_SEGMENT->segment_num;
     int current_gs_register = old_gs_register;
     long j;
@@ -230,6 +258,7 @@
 }
 
 
+
 /************************************************************/
 /*  Algorithm for regular (non-light) finalizers.
     Follows closely pypy/doc/discussion/finalizer-order.rst
@@ -331,17 +360,19 @@
     struct list_s *lst = f->objects_with_finalizers;
     long i, count = list_count(lst);
     lst->count = 0;
-    f->count_non_young = 0;
 
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(lst, i);
+        uintptr_t qindex = list_item(lst, i + 1);
 
         assert(_finalization_state(x) != 1);
         if (_finalization_state(x) >= 2) {
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
             continue;
         }
         LIST_APPEND(marked, x);
+        LIST_APPEND(marked, qindex);
 
         struct list_s *pending = _finalizer_pending;
         LIST_APPEND(pending, x);
@@ -373,27 +404,29 @@
     struct list_s *run_finalizers = f->run_finalizers;
 
     long i, count = list_count(marked);
-    for (i = 0; i < count; i++) {
+    for (i = 0; i < count; i += 2) {
         object_t *x = (object_t *)list_item(marked, i);
+        uintptr_t qindex = list_item(marked, i + 1);
 
         int state = _finalization_state(x);
         assert(state >= 2);
         if (state == 2) {
-            if (run_finalizers == NULL)
-                run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
+            LIST_APPEND(run_finalizers, qindex);
             _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
             list_set_item(lst, lst->count++, (uintptr_t)x);
+            list_set_item(lst, lst->count++, qindex);
         }
     }
-    list_free(marked);
+    LIST_FREE(marked);
 
     f->run_finalizers = run_finalizers;
 }
 
+
 static void deal_with_objects_with_finalizers(void)
 {
     /* for non-light finalizers */
@@ -436,11 +469,10 @@
 static void mark_visit_from_finalizer1(
     struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
-    if (f != NULL && f->run_finalizers != NULL) {
-        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
-                       ({
-                           mark_visit_possibly_overflow_object(item, pseg);
-                       }));
+    long i, count = list_count(f->run_finalizers);
+    for (i = 0; i < count; i += 2) {
+        object_t *x = (object_t *)list_item(f->run_finalizers, i);
+        mark_visit_possibly_overflow_object(x, pseg);
     }
 }
 
@@ -454,40 +486,6 @@
     mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
-static void _execute_finalizers(struct finalizers_s *f)
-{
-    if (f->run_finalizers == NULL)
-        return;   /* nothing to do */
-
- restart:
-    if (f->running_next != NULL)
-        return;   /* in a nested invocation of execute_finalizers() */
-
-    uintptr_t next = 0, total = list_count(f->run_finalizers);
-    f->running_next = &next;
-
-    while (next < total) {
-        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
-        list_set_item(f->run_finalizers, next, 0);
-        next++;
-
-        stmcb_finalizer(obj);
-    }
-    if (next == (uintptr_t)-1) {
-        /* transaction committed: the whole 'f' was freed */
-        return;
-    }
-    f->running_next = NULL;
-
-    if (f->run_finalizers->count > total) {
-        memmove(f->run_finalizers->items,
-                f->run_finalizers->items + total,
-                (f->run_finalizers->count - total) * sizeof(uintptr_t));
-        goto restart;
-    }
-
-    LIST_FREE(f->run_finalizers);
-}
 
 /* XXX: according to translator.backendopt.finalizer, getfield_gc
         for primitive types is a safe op in light finalizers.
@@ -495,43 +493,107 @@
         getfield on *dying obj*).
 */
 
+
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
-    /* called between transactions */
-    rewind_jmp_buf rjbuf;
-    stm_rewind_jmp_enterframe(tl, &rjbuf);
-    _stm_start_transaction(tl);
-    /* XXX: become inevitable, bc. otherwise, we would need to keep
-       around the original g_finalizers.run_finalizers to restore it
-       in case of an abort. */
-    _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
-    /* did it work? */
-    if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
-        /* avoid blocking here, waiting for another INEV transaction.
-           If we did that, application code could not proceed (start the
-           next transaction) and it will not be obvious from the profile
-           why we were WAITing. */
-        _stm_commit_transaction();
-        stm_rewind_jmp_leaveframe(tl, &rjbuf);
+    /* called between transactions
+     * run old-style finalizers (q_index=-1) and run triggers for all finalizer
+     * queues that are not empty. */
+    dprintf(("invoke_general_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
+    if (list_is_empty(g_finalizers.run_finalizers))
         return;
-    }
+
+    bool *to_trigger = calloc(g_finalizer_triggers.count + 1, sizeof(bool));
 
     while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
         /* somebody is adding more finalizers (_commit_finalizer()) */
         stm_spin_loop();
     }
-    struct finalizers_s copy = g_finalizers;
-    assert(copy.running_next == NULL);
-    g_finalizers.run_finalizers = NULL;
-    /* others may add to g_finalizers again: */
+
+    int count = list_count(g_finalizers.run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(g_finalizers.run_finalizers, i + 1);
+        to_trigger[qindex + 1] = true; // also for -1!
+    }
+
     __sync_lock_release(&g_finalizers.lock);
 
-    if (copy.run_finalizers != NULL) {
-        _execute_finalizers(&copy);
+    // trigger now:
+    for (int i = 1; i < g_finalizer_triggers.count; i++) {
+        if (to_trigger[i]) {
+            dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
+            // XXX: check that triggers *really* cannot touch GC-memory,
+            // otherwise, this needs to run in an (inevitable) transaction
+            g_finalizer_triggers.triggers[i - 1]();
+        }
+    }
+
+    if (!to_trigger[0]) {
+        // 0 is the qindex for old-style finalizers,
+        // so nothing todo here.
+        free(to_trigger);
+        return;
+    }
+    free(to_trigger);
+
+    // run old-style finalizers:
+    dprintf(("invoke-oldstyle-finalizers\n"));
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+    _stm_start_transaction(tl);
+
+    object_t *obj;
+    while ((obj = stm_next_to_finalize(-1)) != NULL) {
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
+        stmcb_finalizer(obj);
     }
 
     _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
+}
 
-    LIST_FREE(copy.run_finalizers);
+object_t* stm_next_to_finalize(int queue_index) {
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        stm_spin_loop();
+    }
+
+    int count = list_count(g_finalizers.run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(g_finalizers.run_finalizers, i + 1);
+        if (qindex == queue_index) {
+            /* XXX: become inevitable, bc. otherwise, we would need to keep
+               around the original g_finalizers.run_finalizers to restore it
+               in case of an abort. */
+            if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {
+                _stm_become_inevitable(MSG_INEV_DONT_SLEEP);
+                /* did it work? */
+                if (STM_PSEGMENT->transaction_state != TS_INEVITABLE) {   /* no */
+                    /* avoid blocking here, waiting for another INEV transaction.
+                       If we did that, application code could not proceed (start the
+                       next transaction) and it will not be obvious from the profile
+                       why we were WAITing. */
+                    __sync_lock_release(&g_finalizers.lock);
+                    return NULL;
+                }
+            }
+
+            /* Remove obj from list and return it. */
+            object_t *obj = (object_t*)list_item(g_finalizers.run_finalizers, i);
+            if (i < count - 2) {
+                memmove(&g_finalizers.run_finalizers->items[i],
+                        &g_finalizers.run_finalizers->items[i + 2],
+                        (count - 2) * sizeof(uintptr_t));
+            }
+            g_finalizers.run_finalizers->count -= 2;
+
+            __sync_lock_release(&g_finalizers.lock);
+            return obj;
+        }
+    }
+
+    /* others may add to g_finalizers again: */
+    __sync_lock_release(&g_finalizers.lock);
+
+    return NULL;
 }
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -1,17 +1,20 @@
+#ifndef _STM_FINALIZER_H_
+#define _STM_FINALIZER_H_
+
 #include <stdint.h>
 
 /* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
     long lock;
     struct list_s *objects_with_finalizers;
-    uintptr_t count_non_young;
+    struct list_s *probably_young_objects_with_finalizers;
     struct list_s *run_finalizers;
     uintptr_t *running_next;
 };
 
 static void mark_visit_from_finalizer_pending(void);
-static void deal_with_young_objects_with_finalizers(void);
-static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_young_objects_with_destructors(void);
+static void deal_with_old_objects_with_destructors(void);
 static void deal_with_objects_with_finalizers(void);
 
 static void setup_finalizer(void);
@@ -28,19 +31,16 @@
 
 /* regular finalizers (objs from already-committed transactions) */
 static struct finalizers_s g_finalizers;
+static struct {
+    int count;
+    stm_finalizer_trigger_fn *triggers;
+} g_finalizer_triggers;
+
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl);
 
 #define invoke_general_finalizers(tl)    do {   \
-    if (g_finalizers.run_finalizers != NULL)    \
-        _invoke_general_finalizers(tl);         \
+     _invoke_general_finalizers(tl);         \
 } while (0)
 
-static void _execute_finalizers(struct finalizers_s *f);
-
-#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
-                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
-#define exec_local_finalizers()  do {                   \
-    if (any_local_finalizers())                         \
-        _execute_finalizers(STM_PSEGMENT->finalizers);  \
-} while (0)
+#endif
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -213,7 +213,6 @@
     }
 
     s_mutex_unlock();
-    exec_local_finalizers();
 }
 
 
@@ -807,9 +806,9 @@
 
     LIST_FREE(marked_objects_to_trace);
 
-    /* weakrefs and execute old light finalizers */
+    /* weakrefs and execute old destructors */
     stm_visit_old_weakrefs();
-    deal_with_old_objects_with_finalizers();
+    deal_with_old_objects_with_destructors();
 
     /* cleanup */
     clean_up_segment_lists();
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -1,3 +1,7 @@
+#ifndef _STM_GCPAGE_H_
+#define _STM_GCPAGE_H_
+
+#include <stdbool.h>
 
 /* Granularity when grabbing more unused pages: take 20 at a time */
 #define GCPAGE_NUM_PAGES   20
@@ -22,3 +26,9 @@
 static void major_collection_with_mutex(void);
 static bool largemalloc_keep_object_at(char *data);   /* for largemalloc.c */
 static bool smallmalloc_keep_object_at(char *data);   /* for smallmalloc.c */
+
+static inline bool mark_visited_test(object_t *obj);
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj);
+static void mark_visit_possibly_overflow_object(object_t *obj, struct stm_priv_segment_info_s *pseg);
+
+#endif
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -17,13 +17,13 @@
 
 static struct list_s *list_create(void) __attribute__((unused));
 
-static inline void list_free(struct list_s *lst)
+static inline void _list_free(struct list_s *lst)
 {
     free(lst);
 }
 
 #define LIST_CREATE(lst)  ((lst) = list_create())
-#define LIST_FREE(lst)  (list_free(lst), (lst) = NULL)
+#define LIST_FREE(lst)  (_list_free(lst), (lst) = NULL)
 
 
 static struct list_s *_list_grow(struct list_s *, uintptr_t);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -3,6 +3,8 @@
 # include "core.h"  // silence flymake
 #endif
 
+#include "finalizer.h"
+
 /************************************************************/
 
 #define NURSERY_START         (FIRST_NURSERY_PAGE * 4096UL)
@@ -56,7 +58,6 @@
 
 /************************************************************/
 static object_t *find_existing_shadow(object_t *obj);
-#define GCWORD_MOVED  ((object_t *) -1)
 #define FLAG_SYNC_LARGE       0x01
 
 
@@ -441,23 +442,26 @@
     }
 }
 
-static void collect_objs_still_young_but_with_finalizers(void)
+static void collect_young_objects_with_finalizers(void)
 {
-    struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
-    uintptr_t i, total = list_count(lst);
+    /* for real finalizers: in a minor collection, all young objs must survive! */
 
-    for (i = STM_PSEGMENT->finalizers->count_non_young; i < total; i++) {
+    struct list_s *lst = STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers;
+    long i, count = list_count(lst);
+    for (i = 0; i < count; i += 2) {
+        object_t *obj = (object_t *)list_item(lst, i);
+        uintptr_t qindex = list_item(lst, i + 1);
 
-        object_t *o = (object_t *)list_item(lst, i);
-        minor_trace_if_young(&o);
+        minor_trace_if_young(&obj);
 
-        /* was not actually movable */
-        assert(o == (object_t *)list_item(lst, i));
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+        LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, qindex);
     }
-    STM_PSEGMENT->finalizers->count_non_young = total;
+    list_clear(lst);
 }
 
 
+
 static void throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
 #pragma push_macro("STM_PSEGMENT")
@@ -555,8 +559,7 @@
 
     collect_roots_in_nursery();
 
-    if (STM_PSEGMENT->finalizers != NULL)
-        collect_objs_still_young_but_with_finalizers();
+    collect_young_objects_with_finalizers();
 
     if (STM_PSEGMENT->active_queues != NULL)
         collect_active_queues();
@@ -568,7 +571,7 @@
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     stm_move_young_weakrefs();
     release_privatization_lock(STM_SEGMENT->segment_num);
-    deal_with_young_objects_with_finalizers();
+    deal_with_young_objects_with_destructors();
 
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -1,3 +1,8 @@
+#ifndef _STM_NURSERY_H_
+#define _STM_NURSERY_H_
+
+#include <stdint.h>
+#include <stdbool.h>
 
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
@@ -44,3 +49,11 @@
 
 #define must_abort()   is_abort(STM_SEGMENT->nursery_end)
 static object_t *find_shadow(object_t *obj);
+
+
+#define GCWORD_MOVED  ((object_t *) -1)
+static inline bool _is_young(object_t *obj);
+static inline struct object_s *mark_loc(object_t *obj);
+static inline bool _is_from_same_transaction(object_t *obj);
+
+#endif
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -147,19 +147,19 @@
     for (i = 0; i < NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pr = get_priv_segment(i);
         assert(list_is_empty(pr->objects_pointing_to_nursery));
-        list_free(pr->objects_pointing_to_nursery);
-        list_free(pr->old_objects_with_cards_set);
-        list_free(pr->modified_old_objects);
+        LIST_FREE(pr->objects_pointing_to_nursery);
+        LIST_FREE(pr->old_objects_with_cards_set);
+        LIST_FREE(pr->modified_old_objects);
         assert(list_is_empty(pr->large_overflow_objects));
-        list_free(pr->large_overflow_objects);
-        list_free(pr->young_weakrefs);
-        list_free(pr->old_weakrefs);
+        LIST_FREE(pr->large_overflow_objects);
+        LIST_FREE(pr->young_weakrefs);
+        LIST_FREE(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
         tree_free(pr->callbacks_on_commit_and_abort[1]);
-        list_free(pr->young_objects_with_destructors);
-        list_free(pr->old_objects_with_destructors);
+        LIST_FREE(pr->young_objects_with_destructors);
+        LIST_FREE(pr->old_objects_with_destructors);
         if (pr->active_queues) tree_free(pr->active_queues);
     }
 
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -311,7 +311,6 @@
     assert(_stm_in_transaction(tl));
     ensure_gs_register(tl->last_associated_segment_num);
     assert(STM_SEGMENT->running_thread == tl);
-    exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -716,7 +716,7 @@
 extern void (*stmcb_destructor)(object_t *);
 void stm_enable_destructor(object_t *);
 
-/* Support for regular finalizers.  Unreachable objects with
+/* XXX: Support for regular finalizers.  Unreachable objects with
    finalizers are kept alive, as well as everything they point to, and
    stmcb_finalizer() is called after the major GC.  If there are
    several objects with finalizers that reference each other in a
@@ -730,8 +730,14 @@
    transaction.  For older objects, the finalizer is called from a
    random thread between regular transactions, in a new custom
    transaction. */
-extern void (*stmcb_finalizer)(object_t *);
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
+typedef void (*stm_finalizer_trigger_fn)(void);
+void (*stmcb_finalizer)(object_t *);
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers);
+void stm_enable_finalizer(int queue_index, object_t *obj);
+
+/* Returns the next object that supposedly died and should have its finalizer
+   called. XXX: This function turns the transaction inevitable. */
+object_t *stm_next_to_finalize(int queue_index);
 
 
 /* dummies for now: */
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -52,7 +52,6 @@
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
 object_t *stm_allocate(ssize_t size_rounded_up);
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
-object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
 object_t *stm_allocate_noconflict(ssize_t size_rounded_up);
 
 /*void stm_write_card(); use _checked_stm_write_card() instead */
@@ -209,7 +208,11 @@
 void (*stmcb_destructor)(object_t *);
 void stm_enable_destructor(object_t *);
 
+typedef void (*stm_finalizer_trigger_fn)(void);
 void (*stmcb_finalizer)(object_t *);
+void stm_setup_finalizer_queues(int number, stm_finalizer_trigger_fn *triggers);
+void stm_enable_finalizer(int queue_index, object_t *obj);
+object_t *stm_next_to_finalize(int queue_index);
 
 typedef struct stm_hashtable_s stm_hashtable_t;
 typedef ... stm_hashtable_entry_t;
@@ -748,15 +751,19 @@
     return o
 
 def stm_allocate_with_finalizer(size):
-    o = lib.stm_allocate_with_finalizer(size)
+    # OLD-Style finalizers!
+    o = lib.stm_allocate(size)
     tid = 42 + size
     lib._set_type_id(o, tid)
+    lib.stm_enable_finalizer(-1, o)
     return o
 
 def stm_allocate_with_finalizer_refs(n):
-    o = lib.stm_allocate_with_finalizer(HDR + n * WORD)
+    # OLD-Style finalizers!
+    o = lib.stm_allocate(HDR + n * WORD)
     tid = 421420 + n
     lib._set_type_id(o, tid)
+    lib.stm_enable_finalizer(-1, o)
     return o
 
 SIZEOF_HASHTABLE_OBJ = 16 + lib.SIZEOF_MYOBJ
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -7,6 +7,14 @@
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger():
+            # triggers not needed for destructors
+            assert False
+        triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
+        lib.stm_setup_finalizer_queues(1, triggers)
+        triggers = None
+        #
         @ffi.callback("void(object_t *)")
         def destructor(obj):
             assert stm_get_obj_size(obj) == 48
@@ -140,13 +148,20 @@
 
 
 
-class TestRegularFinalizer(BaseTest):
+class TestOldStyleRegularFinalizer(BaseTest):
     expect_content_character = None
     run_major_collect_in_finalizer = False
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
         #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger():
+            # triggers not needed for oldstyle-finalizer tests
+            assert False
+        triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
+        lib.stm_setup_finalizer_queues(1, triggers)
+        #
         @ffi.callback("void(object_t *)")
         def finalizer(obj):
             print "finalizing!", obj
@@ -161,19 +176,24 @@
         self._finalizer_keepalive = finalizer
 
     def expect_finalized(self, objs):
-        assert self.finalizers_called == objs
+        if isinstance(objs, int):
+            assert len(self.finalizers_called) == objs
+        else:
+            assert self.finalizers_called == objs
         self.finalizers_called = []
 
     def test_no_finalizer(self):
         self.start_transaction()
         lp1 = stm_allocate(48)
         stm_major_collect()
+        self.commit_transaction()
         self.expect_finalized([])
 
     def test_no_finalizer_in_minor_collection(self):
         self.start_transaction()
         lp1 = stm_allocate_with_finalizer(48)
         stm_minor_collect()
+        self.commit_transaction()
         self.expect_finalized([])
 
     def test_finalizer_in_major_collection(self):
@@ -182,10 +202,17 @@
             lp1 = stm_allocate_with_finalizer(48)
             lp2 = stm_allocate_with_finalizer(48)
             lp3 = stm_allocate_with_finalizer(48)
+            self.expect_finalized([])
+            self.push_roots([lp1, lp2, lp3])
+            self.commit_transaction()  # move finalizer-objs to global queue
+            self.start_transaction()
+            lp1, lp2, lp3 = self.pop_roots()
             print repeat, lp1, lp2, lp3
-            self.expect_finalized([])
             stm_major_collect()
+            self.commit_transaction()  # invoke finalizers
             self.expect_finalized([lp1, lp2, lp3])
+            self.start_transaction()
+        self.commit_transaction()
 
     def test_finalizer_from_other_thread(self):
         self.start_transaction()
@@ -200,6 +227,7 @@
         self.expect_finalized([])      # marked as dead, but wrong thread
         #
         self.switch(0)
+        py.test.xfail("we don't finalize in the same transaction anymore.")
         self.expect_finalized([lp1])   # now it has been finalized
 
     def test_finalizer_ordering(self):
@@ -210,7 +238,14 @@
         print lp1, lp2, lp3
         stm_set_ref(lp3, 0, lp1)
         stm_set_ref(lp1, 0, lp2)
+
+        self.push_roots([lp1, lp2, lp3])
+        self.commit_transaction()  # move finalizer-objs to global queue
+        self.start_transaction()
+        lp1, lp2, lp3 = self.pop_roots()
+
         stm_major_collect()
+        self.commit_transaction() # invoke finalizers
         self.expect_finalized([lp3])
 
     def test_finalizer_extra_transaction(self):
@@ -222,16 +257,16 @@
 
         self.start_transaction()
         lp1b = self.pop_root()
-        assert lp1b == lp1
+        # assert lp1b == lp1 <- lp1 can be in nursery now
         self.expect_finalized([])
-        self.commit_transaction()
+        self.commit_transaction() # finalizer-obj moved to global queue
         self.expect_finalized([])
 
         self.start_transaction()
         stm_major_collect()
         self.expect_finalized([])
-        self.commit_transaction()
-        self.expect_finalized([lp1])
+        self.commit_transaction() # invoke finalizers
+        self.expect_finalized([lp1b])
 
     def test_run_cb_for_all_threads(self):
         self.start_transaction()
@@ -245,8 +280,11 @@
 
         self.expect_finalized([])
         stm_major_collect()
+        self.commit_transaction()
+        self.expect_finalized(1)
         self.switch(0)
-        self.expect_finalized([lp2, lp1])
+        self.commit_transaction()
+        self.expect_finalized(1)
 
     def test_run_major_collect_in_finalizer(self):
         self.run_major_collect_in_finalizer = True
@@ -256,6 +294,7 @@
         lp3 = stm_allocate_with_finalizer(32)
         print lp1, lp2, lp3
         stm_major_collect()
+        self.commit_transaction()
 
     def test_new_objects_w_finalizers(self):
         self.switch(2)
@@ -272,4 +311,5 @@
 
         self.expect_finalized([])
         stm_major_collect()
+        self.commit_transaction()
         self.expect_finalized([lp1])
diff --git a/c8/test/test_list.py b/c8/test/test_list.py
--- a/c8/test/test_list.py
+++ b/c8/test/test_list.py
@@ -6,7 +6,7 @@
 ffi = cffi.FFI()
 ffi.cdef("""
 struct list_s *list_create(void);
-void list_free(struct list_s *lst);
+void _list_free(struct list_s *lst);
 struct list_s *list_append(struct list_s *lst, uintptr_t item);
 uintptr_t list_count(struct list_s *lst);
 uintptr_t list_item(struct list_s *lst, uintptr_t index);
@@ -144,5 +144,5 @@
     assert lib.list_count(a) == 17 + 7
     for i, expected in enumerate(range(103, 120) + range(113, 120)):
         assert lib.list_item(a, i) == expected
-    lib.list_free(b)
-    lib.list_free(a)
+    lib._list_free(b)
+    lib._list_free(a)

From pypy.commits at gmail.com  Thu Nov 24 05:00:25 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:25 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: write some tests for
 finalizer queues
Message-ID: <5836ba39.ca57c20a.49dfd.b0a2@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r2001:26622312cfa8
Date: 2016-11-17 11:37 +0100
http://bitbucket.org/pypy/stmgc/changeset/26622312cfa8/

Log:	write some tests for finalizer queues

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -528,9 +528,13 @@
             // XXX: check that triggers *really* cannot touch GC-memory,
             // otherwise, this needs to run in an (inevitable) transaction
 #ifndef NDEBUG
+            char *old_gs_register = STM_SEGMENT->segment_base;
             set_gs_register(NULL);
 #endif
             g_finalizer_triggers.triggers[i]();
+#ifndef NDEBUG
+            set_gs_register(old_gs_register);
+#endif
         }
     }
 
@@ -586,6 +590,8 @@
 }
 
 object_t* stm_next_to_finalize(int queue_index) {
+    assert(STM_PSEGMENT->transaction_state != TS_NONE);
+
     /* first check local run_finalizers queue, then global */
     if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
         struct list_s *lst = STM_PSEGMENT->finalizers->run_finalizers;
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -11,6 +11,7 @@
         def trigger():
             # triggers not needed for destructors
             assert False
+        self._trigger_keepalive = trigger
         triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
         lib.stm_setup_finalizer_queues(1, triggers)
         triggers = None
@@ -147,6 +148,190 @@
         self.expect_finalized([lp1], from_tlnum=0)
 
 
+class TestRegularFinalizerQueues(BaseTest):
+    expect_content_character = None
+    run_major_collect_in_finalizer = False
+
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger0():
+            self.queues_triggered.append(0)
+        @ffi.callback("stm_finalizer_trigger_fn")
+        def trigger1():
+            self.queues_triggered.append(1)
+        self._trigger_keepalive = [trigger0, trigger1]
+        triggers = ffi.new("stm_finalizer_trigger_fn[]", self._trigger_keepalive)
+        lib.stm_setup_finalizer_queues(2, triggers)
+        #
+        self.queues_triggered = []
+
+    def get_queues_triggered(self):
+        res = set(self.queues_triggered)
+        self.queues_triggered = []
+        return res
+
+    def get_to_finalize(self, qindex):
+        obj = lib.stm_next_to_finalize(qindex)
+        res = []
+        while obj:
+            res.append(obj)
+            obj = lib.stm_next_to_finalize(qindex)
+        return res
+
+
+
+    def test_no_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        stm_major_collect()
+        assert not self.get_queues_triggered()
+
+    def test_no_finalizer_in_minor_collection(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        stm_minor_collect()
+        assert not self.get_queues_triggered()
+
+    def test_finalizer_in_major_collection(self):
+        self.start_transaction()
+        for repeat in range(4):
+            lp1 = stm_allocate(48)
+            lp2 = stm_allocate(48)
+            lp3 = stm_allocate(48)
+            if repeat % 2 == 0: # register on young
+                lib.stm_enable_finalizer(0, lp1)
+                lib.stm_enable_finalizer(1, lp2)
+                lib.stm_enable_finalizer(1, lp3)
+            assert not self.get_queues_triggered()
+            self.push_roots([lp1, lp2, lp3])
+            stm_minor_collect()
+            lp1, lp2, lp3 = self.pop_roots()
+            if repeat % 2 == 1: # register on old
+                lib.stm_enable_finalizer(0, lp1)
+                lib.stm_enable_finalizer(1, lp2)
+                lib.stm_enable_finalizer(1, lp3)
+            print repeat, lp1, lp2, lp3
+            stm_major_collect()
+            assert {0, 1} == self.get_queues_triggered()
+            assert [lp1] == self.get_to_finalize(0)
+            assert [lp2, lp3] == self.get_to_finalize(1)
+
+    def test_finalizer_from_other_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        stm_set_char(lp1, 'H')
+        self.expect_content_character = 'H'
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_major_collect()
+        assert not self.get_queues_triggered()
+        #
+        self.switch(0)
+        assert {0} == self.get_queues_triggered()
+        assert [lp1] == self.get_to_finalize(0)
+
+    def test_finalizer_ordering(self):
+        self.start_transaction()
+        lp1 = stm_allocate_refs(1)
+        lp2 = stm_allocate_refs(1)
+        lp3 = stm_allocate_refs(1)
+        lib.stm_enable_finalizer(1, lp1)
+        lib.stm_enable_finalizer(1, lp2)
+        lib.stm_enable_finalizer(1, lp3)
+
+        print lp1, lp2, lp3
+        stm_set_ref(lp3, 0, lp1)
+        stm_set_ref(lp1, 0, lp2)
+
+        self.push_roots([lp1, lp2, lp3])
+        stm_minor_collect()
+        lp1, lp2, lp3 = self.pop_roots()
+
+        stm_major_collect()
+        assert {1} == self.get_queues_triggered()
+        assert [lp3] == self.get_to_finalize(1)
+
+    def test_finalizer_extra_transaction(self):
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        lib.stm_enable_finalizer(0, lp1)
+        self.push_root(lp1)
+        self.commit_transaction()
+
+        self.start_transaction()
+        lp1b = self.pop_root()
+        # assert lp1b == lp1 <- lp1 can be in nursery now
+        assert not self.get_queues_triggered()
+        self.commit_transaction() # finalizer-obj moved to global queue
+        assert not self.get_queues_triggered()
+
+        self.start_transaction()
+        stm_major_collect()
+        assert not self.get_queues_triggered()
+        self.commit_transaction() # invoke finalizers
+        self.start_transaction()
+        assert {0} == self.get_queues_triggered()
+        assert [lp1b] == self.get_to_finalize(0)
+
+    def test_run_cb_for_all_threads(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_finalizer(0, lp1)
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        lp2 = stm_allocate(56)
+        lib.stm_enable_finalizer(1, lp2)
+        print lp2
+
+        assert not self.get_queues_triggered()
+        stm_major_collect()
+        assert len(self.get_to_finalize(1)) == 1
+        self.switch(0)
+        assert self.get_queues_triggered() == {0, 1}
+        assert len(self.get_to_finalize(0)) == 1
+
+    def test_run_major_collect_in_finalizer(self):
+        self.run_major_collect_in_finalizer = True
+        self.start_transaction()
+        lp1 = stm_allocate(32)
+        lp2 = stm_allocate(32)
+        lp3 = stm_allocate(32)
+        lib.stm_enable_finalizer(0, lp1)
+        lib.stm_enable_finalizer(0, lp2)
+        lib.stm_enable_finalizer(0, lp3)
+        print lp1, lp2, lp3
+        stm_major_collect()
+
+    def test_new_objects_w_finalizers(self):
+        self.switch(2)
+        self.start_transaction()
+        lp1 = stm_allocate_refs(3)
+        lp2 = stm_allocate_refs(3)
+        lib.stm_enable_finalizer(0, lp1)
+        lib.stm_enable_finalizer(0, lp2)
+        stm_set_ref(lp1, 0, lp2)
+
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        lp2 = stm_get_ref(lp1, 0)
+        # lp1, lp2 have WB_EXECUTED objs
+
+        assert not self.get_queues_triggered()
+        stm_major_collect()
+        assert [lp1] == self.get_to_finalize(0)
+
 
 class TestOldStyleRegularFinalizer(BaseTest):
     expect_content_character = None
@@ -159,6 +344,7 @@
         def trigger():
             # triggers not needed for oldstyle-finalizer tests
             assert False
+        self._trigger_keepalive = trigger
         triggers = ffi.new("stm_finalizer_trigger_fn*", trigger)
         lib.stm_setup_finalizer_queues(1, triggers)
         #

From pypy.commits at gmail.com  Thu Nov 24 05:00:26 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:26 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: add finalizers to test_random
Message-ID: <5836ba3a.c5311c0a.515e7.80ba@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r2002:dbfc15bd706f
Date: 2016-11-17 13:34 +0100
http://bitbucket.org/pypy/stmgc/changeset/dbfc15bd706f/

Log:	add finalizers to test_random

diff --git a/c8/test/test_zrandom.py b/c8/test/test_zrandom.py
--- a/c8/test/test_zrandom.py
+++ b/c8/test/test_zrandom.py
@@ -411,13 +411,20 @@
     global_state.add_noconfl_obj(r)
 
 def op_allocate_ref(ex, global_state, thread_state):
-    num = str(global_state.rnd.randrange(1, 1000))
+    g_rnd = global_state.rnd
+    num = str(g_rnd.randrange(1, 1000))
     r = global_state.get_new_root_name(True, num)
     thread_state.push_roots(ex)
     ex.do('%s = stm_allocate_refs(%s)' % (r, num))
     ex.do('# 0x%x' % (int(ffi.cast("uintptr_t", ex.content[r]))))
     thread_state.transaction_state.add_root(r, "ffi.NULL", True)
 
+    if g_rnd.randrange(10) == 0:
+        # XXX: should empty the queues sometimes! but calling next_to_finalize
+        # *may* turn inevitable (but only if there is something in the queue).
+        # so that's a bit involved...
+        ex.do('lib.stm_enable_finalizer(%s, %s)' % (g_rnd.randrange(2), r))
+
     thread_state.pop_roots(ex)
     thread_state.reload_roots(ex)
     thread_state.register_root(r)
@@ -585,6 +592,25 @@
         N_THREADS = self.NB_THREADS
         ex = Exec(self)
         ex.do("################################################################\n"*10)
+        ex.do("""
+
+ at ffi.callback("stm_finalizer_trigger_fn")
+def trigger0():
+    pass
+ at ffi.callback("stm_finalizer_trigger_fn")
+def trigger1():
+    pass
+self._trigger_keepalive = [trigger0, trigger1]
+triggers = ffi.new("stm_finalizer_trigger_fn[]", self._trigger_keepalive)
+lib.stm_setup_finalizer_queues(2, triggers)
+#
+ at ffi.callback("void(object_t *)")
+def finalizer(obj):
+    pass
+lib.stmcb_finalizer = finalizer
+self._finalizer_keepalive = finalizer
+        """)
+
         ex.do('# initialization')
 
         global_state = GlobalState(ex, rnd)
@@ -624,7 +650,7 @@
         ]
         possible_actions = [item for sublist in possible_actions for item in sublist]
         print possible_actions
-        for _ in range(2000):
+        for _ in range(3000):
             # make sure we are in a transaction:
             curr_thread = op_switch_thread(ex, global_state, curr_thread)
 

From pypy.commits at gmail.com  Thu Nov 24 05:00:28 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:28 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: allow to run arbitrary things
 in finalizer triggers
Message-ID: <5836ba3c.e626c20a.f98d5.bd0c@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r2003:7e7619989b07
Date: 2016-11-21 14:41 +0100
http://bitbucket.org/pypy/stmgc/changeset/7e7619989b07/

Log:	allow to run arbitrary things in finalizer triggers

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -17,8 +17,8 @@
     f->objects_with_finalizers = list_create();
     f->probably_young_objects_with_finalizers = list_create();
     f->run_finalizers = list_create();
-    f->running_next = NULL;
     f->lock = 0;
+    f->running_trigger_now = NULL;
 }
 
 static void setup_finalizer(void)
@@ -75,16 +75,10 @@
            'g_finalizers.run_finalizers', dropping any initial NULLs
            (finalizers already called) */
         struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
-        uintptr_t frm = 0;
-        if (STM_PSEGMENT->finalizers->running_next != NULL) {
-            frm = *STM_PSEGMENT->finalizers->running_next;
-            assert(frm <= list_count(src));
-            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
-        }
-        if (frm < list_count(src)) {
+        if (list_count(src)) {
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
-                src, frm);
+                src, 0);
         }
     }
     LIST_FREE(STM_PSEGMENT->finalizers->run_finalizers);
@@ -108,15 +102,16 @@
 {
     /* like _commit_finalizers(), but forget everything from the
        current transaction */
-    if (pseg->finalizers->running_next != NULL) {
-        *pseg->finalizers->running_next = (uintptr_t)-1;
-    }
     LIST_FREE(pseg->finalizers->run_finalizers);
     LIST_FREE(pseg->finalizers->objects_with_finalizers);
     LIST_FREE(pseg->finalizers->probably_young_objects_with_finalizers);
     // re-init
     init_finalizers(pseg->finalizers);
 
+    // if we were running triggers, release the lock:
+    if (g_finalizers.running_trigger_now == pseg)
+        g_finalizers.running_trigger_now = NULL;
+
     /* call the light finalizers for objects that are about to
        be forgotten from the current transaction */
     char *old_gs_register = STM_SEGMENT->segment_base;
@@ -494,15 +489,17 @@
         getfield on *dying obj*).
 */
 
-static bool _trigger_finalizer_queues(struct finalizers_s *f)
+static void _trigger_finalizer_queues(struct finalizers_s *f)
 {
     /* runs triggers of finalizer queues that have elements in the queue. May
-       run outside of a transaction, as triggers should be safe to call(?)
+       NOT run outside of a transaction, but triggers never leave the
+       transactional zone.
 
-       returns true if there are old-style finalizers to run */
+       returns true if there are also old-style finalizers to run */
+    assert(in_transaction(STM_PSEGMENT->pub.running_thread));
 
-    bool trigger_oldstyle = false;
-    bool *to_trigger = calloc(g_finalizer_triggers.count, sizeof(bool));
+    bool *to_trigger = (bool*)alloca(g_finalizer_triggers.count * sizeof(bool));
+    memset(to_trigger, 0, g_finalizer_triggers.count * sizeof(bool));
 
     while (__sync_lock_test_and_set(&f->lock, 1) != 0) {
         /* somebody is adding more finalizers (_commit_finalizer()) */
@@ -513,10 +510,7 @@
     for (int i = 0; i < count; i += 2) {
         int qindex = (int)list_item(f->run_finalizers, i + 1);
         dprintf(("qindex=%d\n", qindex));
-        if (qindex == -1)
-            trigger_oldstyle = true;
-        else
-            to_trigger[qindex] = true;
+        to_trigger[qindex] = true;
     }
 
     __sync_lock_release(&f->lock);
@@ -525,22 +519,20 @@
     for (int i = 0; i < g_finalizer_triggers.count; i++) {
         if (to_trigger[i]) {
             dprintf(("invoke-finalizer-trigger(qindex=%d)\n", i));
-            // XXX: check that triggers *really* cannot touch GC-memory,
-            // otherwise, this needs to run in an (inevitable) transaction
-#ifndef NDEBUG
-            char *old_gs_register = STM_SEGMENT->segment_base;
-            set_gs_register(NULL);
-#endif
             g_finalizer_triggers.triggers[i]();
-#ifndef NDEBUG
-            set_gs_register(old_gs_register);
-#endif
         }
     }
+}
 
-    free(to_trigger);
-
-    return trigger_oldstyle;
+static bool _has_oldstyle_finalizers(struct finalizers_s *f)
+{
+    int count = list_count(f->run_finalizers);
+    for (int i = 0; i < count; i += 2) {
+        int qindex = (int)list_item(f->run_finalizers, i + 1);
+        if (qindex == -1)
+            return true;
+    }
+    return false;
 }
 
 static void _invoke_local_finalizers()
@@ -548,10 +540,27 @@
     /* called inside a transaction; invoke local triggers, process old-style
      * local finalizers */
     dprintf(("invoke_local_finalizers %lu\n", list_count(STM_PSEGMENT->finalizers->run_finalizers)));
-    if (list_is_empty(STM_PSEGMENT->finalizers->run_finalizers))
+    if (list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)
+        && list_is_empty(g_finalizers.run_finalizers))
         return;
 
-    if (!_trigger_finalizer_queues(STM_PSEGMENT->finalizers))
+    struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
+    //try to run local triggers
+    if (STM_PSEGMENT->finalizers->running_trigger_now == NULL) {
+        // we are not recursively running them
+        STM_PSEGMENT->finalizers->running_trigger_now = pseg;
+        _trigger_finalizer_queues(STM_PSEGMENT->finalizers);
+        STM_PSEGMENT->finalizers->running_trigger_now = NULL;
+    }
+
+    // try to run global triggers
+    if (__sync_lock_test_and_set(&g_finalizers.running_trigger_now, pseg) == NULL) {
+        // nobody is already running these triggers (recursively)
+        _trigger_finalizer_queues(&g_finalizers);
+        g_finalizers.running_trigger_now = NULL;
+    }
+
+    if (!_has_oldstyle_finalizers(STM_PSEGMENT->finalizers))
         return; // no oldstyle to run
 
     object_t *obj;
@@ -560,25 +569,25 @@
     }
 }
 
-
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
     /* called between transactions
-     * run old-style finalizers (q_index=-1) and run triggers for all finalizer
+     * triggers not called here, since all should have been called already in _invoke_local_finalizers!
+     * run old-style finalizers (q_index=-1)
      * queues that are not empty. */
     dprintf(("invoke_general_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
     if (list_is_empty(g_finalizers.run_finalizers))
         return;
 
-    if (!_trigger_finalizer_queues(&g_finalizers))
-        return; // no oldstyle finalizers to run
+    if (!_has_oldstyle_finalizers(&g_finalizers))
+        return; // no oldstyle to run
 
     // run old-style finalizers:
-    dprintf(("invoke-oldstyle-finalizers\n"));
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
 
+    dprintf(("invoke_oldstyle_finalizers %lu\n", list_count(g_finalizers.run_finalizers)));
     object_t *obj;
     while ((obj = stm_next_to_finalize(-1)) != NULL) {
         assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -6,10 +6,10 @@
 /* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
     long lock;
+    struct stm_priv_segment_info_s * running_trigger_now; /* our PSEG, if we are running triggers */
     struct list_s *objects_with_finalizers;
-    struct list_s *probably_young_objects_with_finalizers;
+    struct list_s *probably_young_objects_with_finalizers; /* empty on g_finalizers! */
     struct list_s *run_finalizers;
-    uintptr_t *running_next;
 };
 
 static void mark_visit_from_finalizer_pending(void);
@@ -46,8 +46,7 @@
 
 
 #define exec_local_finalizers()  do {                   \
-    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) \
-        _invoke_local_finalizers();  \
+     _invoke_local_finalizers();                     \
 } while (0)
 
 #endif
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -150,7 +150,7 @@
 
 class TestRegularFinalizerQueues(BaseTest):
     expect_content_character = None
-    run_major_collect_in_finalizer = False
+    run_major_collect_in_trigger = False
 
     def setup_method(self, meth):
         BaseTest.setup_method(self, meth)
@@ -158,9 +158,13 @@
         @ffi.callback("stm_finalizer_trigger_fn")
         def trigger0():
             self.queues_triggered.append(0)
+            if self.run_major_collect_in_trigger:
+                stm_major_collect()
         @ffi.callback("stm_finalizer_trigger_fn")
         def trigger1():
             self.queues_triggered.append(1)
+            if self.run_major_collect_in_trigger:
+                stm_major_collect()
         self._trigger_keepalive = [trigger0, trigger1]
         triggers = ffi.new("stm_finalizer_trigger_fn[]", self._trigger_keepalive)
         lib.stm_setup_finalizer_queues(2, triggers)
@@ -276,8 +280,8 @@
 
         self.start_transaction()
         stm_major_collect()
-        assert not self.get_queues_triggered()
-        self.commit_transaction() # invoke finalizers
+        assert {0} == self.get_queues_triggered()
+        self.commit_transaction() # invoke finalizers (triggers again...)
         self.start_transaction()
         assert {0} == self.get_queues_triggered()
         assert [lp1b] == self.get_to_finalize(0)
@@ -301,17 +305,19 @@
         assert self.get_queues_triggered() == {0, 1}
         assert len(self.get_to_finalize(0)) == 1
 
-    def test_run_major_collect_in_finalizer(self):
-        self.run_major_collect_in_finalizer = True
+    def test_run_major_collect_in_trigger(self):
+        self.run_major_collect_in_trigger  = True
         self.start_transaction()
         lp1 = stm_allocate(32)
         lp2 = stm_allocate(32)
         lp3 = stm_allocate(32)
         lib.stm_enable_finalizer(0, lp1)
-        lib.stm_enable_finalizer(0, lp2)
-        lib.stm_enable_finalizer(0, lp3)
+        lib.stm_enable_finalizer(1, lp2)
+        lib.stm_enable_finalizer(1, lp3)
         print lp1, lp2, lp3
         stm_major_collect()
+        assert 1 == len(self.get_to_finalize(0))
+        assert 2 == len(self.get_to_finalize(1))
 
     def test_new_objects_w_finalizers(self):
         self.switch(2)

From pypy.commits at gmail.com  Thu Nov 24 05:00:30 2016
From: pypy.commits at gmail.com (Raemi)
Date: Thu, 24 Nov 2016 02:00:30 -0800 (PST)
Subject: [pypy-commit] stmgc finalizer-queues: fix and a bit of cleanup
Message-ID: <5836ba3e.c64bc20a.a1c99.b1f2@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: finalizer-queues
Changeset: r2004:0bd420ee56c1
Date: 2016-11-23 17:05 +0100
http://bitbucket.org/pypy/stmgc/changeset/0bd420ee56c1/

Log:	fix and a bit of cleanup

	why are simple things so hard to get right?

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -70,30 +70,31 @@
         stm_spin_loop();
     }
 
-    if (!list_is_empty(STM_PSEGMENT->finalizers->run_finalizers)) {
+    struct finalizers_s *local_fs = STM_PSEGMENT->finalizers;
+    if (!list_is_empty(local_fs->run_finalizers)) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
            'g_finalizers.run_finalizers', dropping any initial NULLs
            (finalizers already called) */
-        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
+        struct list_s *src = local_fs->run_finalizers;
         if (list_count(src)) {
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
                 src, 0);
         }
     }
-    LIST_FREE(STM_PSEGMENT->finalizers->run_finalizers);
+    LIST_FREE(local_fs->run_finalizers);
 
     /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
        into 'g_finalizers.objects_with_finalizers' */
     g_finalizers.objects_with_finalizers = list_extend(
         g_finalizers.objects_with_finalizers,
-        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
-    assert(list_is_empty(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers));
-    LIST_FREE(STM_PSEGMENT->finalizers->objects_with_finalizers);
-    LIST_FREE(STM_PSEGMENT->finalizers->probably_young_objects_with_finalizers);
+        local_fs->objects_with_finalizers, 0);
+    LIST_FREE(local_fs->objects_with_finalizers);
+    assert(list_is_empty(local_fs->probably_young_objects_with_finalizers));
+    LIST_FREE(local_fs->probably_young_objects_with_finalizers);
 
     // re-init
-    init_finalizers(STM_PSEGMENT->finalizers);
+    init_finalizers(local_fs);
 
     __sync_lock_release(&g_finalizers.lock);
 }
@@ -219,6 +220,7 @@
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
 
+        assert(list_is_empty(pseg->young_objects_with_destructors));
         struct list_s *lst = pseg->old_objects_with_destructors;
         long i, count = list_count(lst);
         lst->count = 0;
@@ -353,6 +355,7 @@
 
     struct list_s *marked = list_create();
 
+    assert(list_is_empty(f->probably_young_objects_with_finalizers));
     struct list_s *lst = f->objects_with_finalizers;
     long i, count = list_count(lst);
     lst->count = 0;
@@ -611,10 +614,11 @@
                 /* no need to become inevitable for local ones */
                 /* Remove obj from list and return it. */
                 object_t *obj = (object_t*)list_item(lst, i);
-                if (i < count - 2) {
+                int remaining = count - i - 2;
+                if (remaining > 0) {
                     memmove(&lst->items[i],
                             &lst->items[i + 2],
-                            (count - 2) * sizeof(uintptr_t));
+                            remaining * sizeof(uintptr_t));
                 }
                 lst->count -= 2;
                 return obj;
@@ -629,9 +633,10 @@
         stm_spin_loop();
     }
 
-    int count = list_count(g_finalizers.run_finalizers);
+    struct list_s *lst = g_finalizers.run_finalizers;
+    int count = list_count(lst);
     for (int i = 0; i < count; i += 2) {
-        int qindex = (int)list_item(g_finalizers.run_finalizers, i + 1);
+        int qindex = (int)list_item(lst, i + 1);
         if (qindex == queue_index) {
             /* XXX: become inevitable, bc. otherwise, we would need to keep
                around the original g_finalizers.run_finalizers to restore it
@@ -643,20 +648,21 @@
                     /* avoid blocking here, waiting for another INEV transaction.
                        If we did that, application code could not proceed (start the
                        next transaction) and it will not be obvious from the profile
-                       why we were WAITing. */
+                       why we were WAITing. XXX: still true? */
                     __sync_lock_release(&g_finalizers.lock);
                     return NULL;
                 }
             }
 
             /* Remove obj from list and return it. */
-            object_t *obj = (object_t*)list_item(g_finalizers.run_finalizers, i);
-            if (i < count - 2) {
-                memmove(&g_finalizers.run_finalizers->items[i],
-                        &g_finalizers.run_finalizers->items[i + 2],
-                        (count - 2) * sizeof(uintptr_t));
+            object_t *obj = (object_t*)list_item(lst, i);
+            int remaining = count - i - 2;
+            if (remaining > 0) {
+                memmove(&lst->items[i],
+                        &lst->items[i + 2],
+                        remaining * sizeof(uintptr_t));
             }
-            g_finalizers.run_finalizers->count -= 2;
+            lst->count -= 2;
 
             __sync_lock_release(&g_finalizers.lock);
             return obj;

From pypy.commits at gmail.com  Thu Nov 24 05:42:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 02:42:18 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Unsure why now,
 but test_zrpy_vmprof crashes because it is missing this method
Message-ID: <5836c40a.44e61c0a.17aa8.871d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88628:bc1b2570fc4a
Date: 2016-11-24 10:49 +0000
http://bitbucket.org/pypy/pypy/changeset/bc1b2570fc4a/

Log:	Unsure why now, but test_zrpy_vmprof crashes because it is missing
	this method

diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -28,6 +28,8 @@
 class FakeWeakCodeObjectList(object):
     def add_handle(self, handle):
         pass
+    def get_all_handles(self):
+        return []
 
 class VMProf(object):
 

From pypy.commits at gmail.com  Thu Nov 24 05:42:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 02:42:20 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: fix test
Message-ID: <5836c40c.42061c0a.1b947.8d9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88629:626ffaa40d6c
Date: 2016-11-24 10:52 +0000
http://bitbucket.org/pypy/pypy/changeset/626ffaa40d6c/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)

From pypy.commits at gmail.com  Thu Nov 24 06:06:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 03:06:28 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: Revert the
 @jit.call_shortcut support (6305cfb3bad2)
Message-ID: <5836c9b4.471ec20a.76ac6.d341@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88630:6d3400d6ed8f
Date: 2016-11-24 11:16 +0000
http://bitbucket.org/pypy/pypy/changeset/6d3400d6ed8f/

Log:	Revert the @jit.call_shortcut support (6305cfb3bad2)

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -11,7 +11,7 @@
 from rpython.jit.backend.llsupport.symbolic import (WORD,
         get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -359,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -607,31 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        descr = op.getdescr()
-        if descr is None:
-            return
-        assert isinstance(descr, CallDescr)
-        effectinfo = descr.get_extra_info()
-        if effectinfo is None or effectinfo.call_shortcut is None:
-            return
-        if op.type == 'r':
-            cond_call_opnum = rop.COND_CALL_VALUE_R
-        elif op.type == 'i':
-            cond_call_opnum = rop.COND_CALL_VALUE_I
-        else:
-            return
-        cs = effectinfo.call_shortcut
-        ptr_box = op.getarg(1 + cs.argnum)
-        if cs.fielddescr is not None:
-            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                           raw=(ptr_box.type == 'i'))
-        else:
-            value_box = ptr_box
-        self.replace_op_with(op, ResOperation(cond_call_opnum,
-                                              [value_box] + op.getarglist(),
-                                              descr=descr))
-
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,8 +1,7 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr, get_call_descr
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -81,21 +80,6 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
-        call_shortcut = CallShortcut(0, tzdescr)
-        effectinfo = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut)
-        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
-            [lltype.Ptr(T)], lltype.Signed,
-            effectinfo)
-        call_shortcut_2 = CallShortcut(0, None)
-        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut_2)
-        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
-            [lltype.Signed], lltype.Signed,
-            effectinfo_2)
-        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -1452,26 +1436,3 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
-
-    def test_handle_call_shortcut(self):
-        self.check_rewrite("""
-            [p0]
-            i1 = call_i(123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """, """
-            [p0]
-            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
-            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """)
-
-    def test_handle_call_shortcut_2(self):
-        self.check_rewrite("""
-            [i0]
-            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """, """
-            [i0]
-            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """)
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,10 +7,9 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection, CallShortcut)
+    EffectInfo, CallInfoCollection)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
-from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -215,7 +214,6 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
-        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -230,12 +228,6 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
-            if hasattr(funcobj, 'graph'):
-                call_shortcut = self.find_call_shortcut(funcobj.graph)
-            if getattr(func, "_call_shortcut_", False):
-                assert call_shortcut is not None, (
-                    "%r: marked as @jit.call_shortcut but shortcut not found"
-                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -250,8 +242,6 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
-                if hasattr(graph.func, '_call_shortcut_'):
-                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -308,7 +298,6 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
-            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -379,76 +368,3 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
-
-    def find_call_shortcut(self, graph):
-        """Identifies graphs that start like this:
-
-           def graph(x, y, z):         def graph(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r: return r
-        """
-        block = graph.startblock
-        operations = block.operations
-        c_fieldname = None
-        if not operations:
-            v_inst = v_result = block.exitswitch
-        else:
-            op = operations[0]
-            if len(op.args) == 0:
-                return
-            if op.opname != 'getfield':  # check for this form:
-                v_inst = op.args[0]      #     if y is not None;
-                v_result = v_inst        #          return y
-            else:
-                operations = operations[1:]
-                [v_inst, c_fieldname] = op.args
-                v_result = op.result
-        if not isinstance(v_inst, Variable):
-            return
-        if v_result.concretetype != graph.getreturnvar().concretetype:
-            return
-        if v_result.concretetype == lltype.Void:
-            return
-        argnum = i = 0
-        while block.inputargs[i] is not v_inst:
-            if block.inputargs[i].concretetype != lltype.Void:
-                argnum += 1
-            i += 1
-        PSTRUCT = v_inst.concretetype
-        v_check = v_result
-        fastcase = True
-        for op in operations:
-            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
-                    and v_check is op.args[0]):
-                v_check = op.result
-            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
-                v_check = op.result
-                fastcase = not fastcase
-            elif (op.opname in ('int_eq', 'int_ne')
-                    and v_check is op.args[0]
-                    and isinstance(op.args[1], Constant)
-                    and op.args[1].value == 0):
-                v_check = op.result
-                if op.opname == 'int_eq':
-                    fastcase = not fastcase
-            else:
-                return
-        if v_check.concretetype is not lltype.Bool:
-            return
-        if block.exitswitch is not v_check:
-            return
-
-        links = [link for link in block.exits if link.exitcase == fastcase]
-        if len(links) != 1:
-            return
-        [link] = links
-        if link.args != [v_result]:
-            return
-        if not link.target.is_final_block():
-            return
-
-        if c_fieldname is not None:
-            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
-        else:
-            fielddescr = None
-        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,8 +117,7 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True,
-                call_shortcut=None):
+                can_collect=True):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -136,8 +135,7 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect,
-               call_shortcut)
+               can_collect)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -192,7 +190,6 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
-        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -278,8 +275,7 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True,
-                                 call_shortcut=None):
+                                 can_collect=True):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -368,8 +364,7 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect,
-                      call_shortcut)
+                      can_collect)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -392,24 +387,6 @@
 
 # ____________________________________________________________
 
-
-class CallShortcut(object):
-    def __init__(self, argnum, fielddescr):
-        self.argnum = argnum
-        self.fielddescr = fielddescr
-
-    def __eq__(self, other):
-        return (isinstance(other, CallShortcut) and
-                self.argnum == other.argnum and
-                self.fielddescr == other.fielddescr)
-    def __ne__(self, other):
-        return not (self == other)
-    def __hash__(self):
-        return hash((self.argnum, self.fielddescr))
-
-# ____________________________________________________________
-
-
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+from rpython.jit.codewriter.effectinfo import EffectInfo
 
 
 class FakePolicy:
@@ -368,121 +368,3 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
-
-def test_find_call_shortcut():
-    class FakeCPU:
-        def fielddescrof(self, TYPE, fieldname):
-            if isinstance(TYPE, lltype.GcStruct):
-                if fieldname == 'inst_foobar':
-                    return 'foobardescr'
-                if fieldname == 'inst_fooref':
-                    return 'foorefdescr'
-            if TYPE == RAW and fieldname == 'x':
-                return 'xdescr'
-            assert False, (TYPE, fieldname)
-    cc = CallControl(FakeCPU())
-
-    class B(object):
-        foobar = 0
-        fooref = None
-
-    def f1(a, b, c):
-        if b.foobar:
-            return b.foobar
-        b.foobar = a + c
-        return b.foobar
-
-    def f2(x, y, z, b):
-        r = b.fooref
-        if r is not None:
-            return r
-        r = b.fooref = B()
-        return r
-
-    class Space(object):
-        def _freeze_(self):
-            return True
-    space = Space()
-
-    def f3(space, b):
-        r = b.foobar
-        if not r:
-            r = b.foobar = 123
-        return r
-
-    def f4(raw):
-        r = raw.x
-        if r != 0:
-            return r
-        raw.x = 123
-        return 123
-    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
-
-    def f5(b):
-        r = b.foobar
-        if r == 0:
-            r = b.foobar = 123
-        return r
-
-    def f6(b):
-        if b is not None:
-            return b
-        return B()
-
-    def f7(c, a):
-        if a:
-            return a
-        return 123
-
-    def b_or_none(c):
-        if c > 15:
-            return B()
-        return None
-
-    def f(a, c):
-        b = B()
-        f1(a, b, c)
-        f2(a, c, a, b)
-        f3(space, b)
-        r = lltype.malloc(RAW, flavor='raw')
-        f4(r)
-        f5(b)
-        f6(b_or_none(c))
-        f7(c, a)
-
-    rtyper = support.annotate(f, [10, 20])
-    f1_graph = rtyper.annotator.translator._graphof(f1)
-    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
-    f2_graph = rtyper.annotator.translator._graphof(f2)
-    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
-    f3_graph = rtyper.annotator.translator._graphof(f3)
-    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
-    f4_graph = rtyper.annotator.translator._graphof(f4)
-    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
-    f5_graph = rtyper.annotator.translator._graphof(f5)
-    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
-    f6_graph = rtyper.annotator.translator._graphof(f6)
-    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
-    f7_graph = rtyper.annotator.translator._graphof(f7)
-    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
-
-def test_cant_find_call_shortcut():
-    from rpython.jit.backend.llgraph.runner import LLGraphCPU
-
-    @jit.dont_look_inside
-    @jit.call_shortcut
-    def f1(n):
-        return n + 17   # no call shortcut found
-
-    def f(n):
-        return f1(n)
-
-    rtyper = support.annotate(f, [1])
-    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
-    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
-    res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
-    call_op = f_graph.startblock.operations[0]
-    assert call_op.opname == 'direct_call'
-    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
-    assert "shortcut not found" in str(e.value)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -257,27 +257,6 @@
     func.oopspec = "jit.not_in_trace()"   # note that 'func' may take arguments
     return func
 
-def call_shortcut(func):
-    """A decorator to ensure that a function has a fast-path.
-    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
-
-    Only useful on functions that the JIT doesn't normally look inside.
-    It still replaces residual calls to that function with inline code
-    that checks for a fast path, and only does the call if not.  For
-    now, graphs made by the following kinds of functions are detected:
-
-           def func(x, y, z):         def func(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r is None:
-               ...                             ...
-                                           return r
-
-    Fast-path detection is always on, but this decorator makes the
-    codewriter complain if it cannot find the promized fast-path.
-    """
-    func._call_shortcut_ = True
-    return func
-
 
 @oopspec("jit.isconstant(value)")
 @specialize.call_location()

From pypy.commits at gmail.com  Thu Nov 24 06:16:44 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 03:16:44 -0800 (PST)
Subject: [pypy-commit] pypy better-PyDict_Next: add failing test
Message-ID: <5836cc1c.c220c20a.7e1c1.d1b6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: better-PyDict_Next
Changeset: r88631:1df198d55844
Date: 2016-11-24 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/1df198d55844/

Log:	add failing test

diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -181,6 +181,27 @@
         raises(OperationError, space.call_method, w_proxy, 'clear')
         assert api.PyDictProxy_Check(w_proxy)
 
+    def test_typedict(self, space, api):
+        py_type = make_ref(space, space.w_type)
+        py_dict = py_type.c_ob_type.c_tp_dict
+        ppos = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
+
+        ppos[0] = 0
+        pkey = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
+        pvalue = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
+        try:
+            w_copy = space.newdict()
+            while api.PyDict_Next(py_dict, ppos, pkey, pvalue):
+                w_key = from_ref(space, pkey[0])
+                w_value = from_ref(space, pvalue[0])
+                space.setitem(w_copy, w_key, w_value)
+        finally:
+            lltype.free(ppos, flavor='raw')
+            lltype.free(pkey, flavor='raw')
+            lltype.free(pvalue, flavor='raw')
+        api.Py_DecRef(py_type) # release borrowed references
+        # do something with w_copy ?
+
 class AppTestDictObject(AppTestCpythonExtensionBase):
     def test_dictproxytype(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Thu Nov 24 06:41:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 03:41:26 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: fix comment (thanks
 cfbolz)
Message-ID: <5836d1e6.849c1c0a.38a6c.a590@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88632:07b9b5f5b593
Date: 2016-11-24 12:40 +0100
http://bitbucket.org/pypy/pypy/changeset/07b9b5f5b593/

Log:	fix comment (thanks cfbolz)

diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1098,7 +1098,7 @@
     'FORCE_TOKEN/0/r',    # historical name; nowadays, returns the jitframe
     'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
     'STRHASH/1/i',        # only reading the .hash field, might be zero so far
-    'UNICODEHASH/1/i',    #  "
+    'UNICODEHASH/1/i',    #     (unless applied on consts, where .hash is forced)
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----

From pypy.commits at gmail.com  Thu Nov 24 06:53:57 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 03:53:57 -0800 (PST)
Subject: [pypy-commit] pypy conditional_call_value_4: close branch,
 ready to merge
Message-ID: <5836d4d5.8b9a1c0a.d3fdb.a7eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: conditional_call_value_4
Changeset: r88633:0dd61a608f32
Date: 2016-11-24 12:51 +0100
http://bitbucket.org/pypy/pypy/changeset/0dd61a608f32/

Log:	close branch, ready to merge


From pypy.commits at gmail.com  Thu Nov 24 06:53:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 03:53:59 -0800 (PST)
Subject: [pypy-commit] pypy default: hg merge conditional_call_value_4
 (arigo, cfbolz reviewing)
Message-ID: <5836d4d7.c9b3c20a.40d2f.d7b4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88634:a7f012406e30
Date: 2016-11-24 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/a7f012406e30/

Log:	hg merge conditional_call_value_4 (arigo, cfbolz reviewing)

	Get rid of call_shortcut (branch conditional_call_value_3) and
	instead make a real jit.conditional_call_elidable(). Implement it on
	all backends. The single use so far is for ll_strhash(), like the
	@call_shortcut before it, but the plan is to use that also in py3.5
	for unicode objects caching the utf8 version.

diff too long, truncating to 2000 out of 2095 lines

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -326,7 +327,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -789,6 +789,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -811,6 +815,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -663,6 +664,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return compute_hash(hlstr(s))
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return compute_hash(hlunicode(u))
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,9 +9,9 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
@@ -347,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -595,33 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
-        descr = op.getdescr()
-        if descr is None:
-            return
-        assert isinstance(descr, CallDescr)
-        effectinfo = descr.get_extra_info()
-        if effectinfo is None or effectinfo.call_shortcut is None:
-            return
-        if op.type == 'r':
-            cond_call_opnum = rop.COND_CALL_VALUE_R
-        elif op.type == 'i':
-            cond_call_opnum = rop.COND_CALL_VALUE_I
-        else:
-            return
-        cs = effectinfo.call_shortcut
-        ptr_box = op.getarg(1 + cs.argnum)
-        if cs.fielddescr is not None:
-            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                           raw=(ptr_box.type == 'i'))
-        else:
-            value_box = ptr_box
-        self.replace_op_with(op, ResOperation(cond_call_opnum,
-                                              [value_box] + op.getarglist(),
-                                              descr=descr))
-
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,8 +1,7 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr, get_call_descr
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -81,21 +80,6 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
-        call_shortcut = CallShortcut(0, tzdescr)
-        effectinfo = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut)
-        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
-            [lltype.Ptr(T)], lltype.Signed,
-            effectinfo)
-        call_shortcut_2 = CallShortcut(0, None)
-        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut_2)
-        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
-            [lltype.Signed], lltype.Signed,
-            effectinfo_2)
-        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -216,7 +200,6 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
-    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1239,6 +1222,10 @@
                        'i3 = gc_load_i(p0,'
                                  '%(strlendescr.offset)s,'
                                  '%(strlendescr.field_size)s)'],
+        [True,  (1,),  'i3 = strhash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(strhashdescr.offset)s,'
+                                 '-%(strhashdescr.field_size)s)'],
         #[False, (1,),  'i3 = unicodelen(p0)' '->'
         #               'i3 = gc_load_i(p0,'
         #                       '%(unicodelendescr.offset)s,'
@@ -1247,7 +1234,10 @@
                        'i3 = gc_load_i(p0,'
                                '%(unicodelendescr.offset)s,'
                                '%(unicodelendescr.field_size)s)'],
-
+        [True,  (1,),  'i3 = unicodehash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(unicodehashdescr.offset)s,'
+                                 '-%(unicodehashdescr.field_size)s)'],
         ## getitem str/unicode
         [True,  (2,4), 'i3 = unicodegetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,'
@@ -1446,26 +1436,3 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
-
-    def test_handle_call_shortcut(self):
-        self.check_rewrite("""
-            [p0]
-            i1 = call_i(123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """, """
-            [p0]
-            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
-            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """)
-
-    def test_handle_call_shortcut_2(self):
-        self.check_rewrite("""
-            [i0]
-            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """, """
-            [i0]
-            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,7 +16,6 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
-    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -656,10 +656,12 @@
     _COND_CALL_SAVE_REGS = [r.r3, r.r4, r.r5, r.r6, r.r12]
 
     def emit_cond_call(self, op, arglocs, regalloc):
+        resloc = arglocs[0]
+        arglocs = arglocs[1:]
+
         fcond = self.guard_success_cc
         self.guard_success_cc = c.cond_none
         assert fcond != c.cond_none
-        fcond = c.negate(fcond)
 
         jmp_adr = self.mc.get_relative_pos()
         self.mc.trap()        # patched later to a 'bc'
@@ -691,7 +693,10 @@
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.bl_abs(cond_call_adr)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  If we have a result, move
+        # it from r2 to its expected location.
+        if resloc is not None:
+            self.mc.mr(resloc.value, r.SCRATCH2.value)
         relative_target = self.mc.currpos() - jmp_adr
         pmc = OverwritingBuilder(self.mc, jmp_adr, 1)
         BI, BO = c.encoding[fcond]
@@ -701,6 +706,9 @@
         # guard_no_exception too
         self.previous_cond_call_jcond = jmp_adr, BI, BO
 
+    emit_cond_call_value_i = emit_cond_call
+    emit_cond_call_value_r = emit_cond_call
+
 
 class FieldOpAssembler(object):
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -315,6 +315,7 @@
         #   * r2 is the gcmap
         #   * the old value of these regs must already be stored in the jitframe
         #   * on exit, all registers are restored from the jitframe
+        #   * the result of the call, if any, is moved to r2
 
         mc = PPCBuilder()
         self.mc = mc
@@ -347,7 +348,11 @@
         # Finish
         self._reload_frame_if_necessary(mc)
 
+        # Move the result, if any, to r2
+        mc.mr(r.SCRATCH2.value, r.r3.value)
+
         mc.mtlr(r.RCS1.value)     # restore LR
+
         self._pop_core_regs_from_jitframe(mc, saved_regs)
         if supports_floats:
             self._pop_fp_regs_from_jitframe(mc)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1038,14 +1038,34 @@
 
     def prepare_cond_call(self, op):
         self.load_condition_into_cc(op.getarg(0))
-        locs = []
+        self.assembler.guard_success_cc = c.negate(
+            self.assembler.guard_success_cc)
+        # ^^^ if arg0==0, we jump over the next block of code (the call)
+        locs = [None]
         # support between 0 and 4 integer arguments
         assert 2 <= op.numargs() <= 2 + 4
         for i in range(1, op.numargs()):
             loc = self.loc(op.getarg(i))
             assert loc.type != FLOAT
             locs.append(loc)
-        return locs
+        return locs     # [None, function, args...]
+
+    def prepare_cond_call_value_i(self, op):
+        x = self.ensure_reg(op.getarg(0))
+        self.load_condition_into_cc(op.getarg(0))
+        self.rm.force_allocate_reg(op, selected_reg=x)   # spilled if survives
+        # ^^^ if arg0!=0, we jump over the next block of code (the call)
+        locs = [x]
+        # support between 0 and 4 integer arguments
+        assert 2 <= op.numargs() <= 2 + 4
+        for i in range(1, op.numargs()):
+            loc = self.loc(op.getarg(i))
+            assert loc.type != FLOAT
+            locs.append(loc)
+        return locs     # [res, function, args...]
+
+    prepare_cond_call_value_r = prepare_cond_call_value_i
+
 
 def notimplemented(self, op):
     msg = '[PPC/regalloc] %s not implemented\n' % op.getopname()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,9 +2448,6 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
-        if not self.cpu.supports_cond_call_value:
-            py.test.skip("missing supports_cond_call_value")
-
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -711,11 +711,6 @@
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
 
-    def filter(self, builder):
-        if not builder.cpu.supports_cond_call_value and \
-           self.opnum == rop.COND_CALL_VALUE_I:
-            raise test_random.CannotProduceOperation
-
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -20,11 +20,6 @@
     r = Random()
     r.jumpahead(piece*99999999)
     OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,7 +16,6 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
-    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.CallTest):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,10 +7,9 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection, CallShortcut)
+    EffectInfo, CallInfoCollection)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
-from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -215,7 +214,6 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
-        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -230,12 +228,6 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
-            if hasattr(funcobj, 'graph'):
-                call_shortcut = self.find_call_shortcut(funcobj.graph)
-            if getattr(func, "_call_shortcut_", False):
-                assert call_shortcut is not None, (
-                    "%r: marked as @jit.call_shortcut but shortcut not found"
-                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -250,8 +242,6 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
-                if hasattr(graph.func, '_call_shortcut_'):
-                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -308,7 +298,6 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
-            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -379,76 +368,3 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
-
-    def find_call_shortcut(self, graph):
-        """Identifies graphs that start like this:
-
-           def graph(x, y, z):         def graph(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r: return r
-        """
-        block = graph.startblock
-        operations = block.operations
-        c_fieldname = None
-        if not operations:
-            v_inst = v_result = block.exitswitch
-        else:
-            op = operations[0]
-            if len(op.args) == 0:
-                return
-            if op.opname != 'getfield':  # check for this form:
-                v_inst = op.args[0]      #     if y is not None;
-                v_result = v_inst        #          return y
-            else:
-                operations = operations[1:]
-                [v_inst, c_fieldname] = op.args
-                v_result = op.result
-        if not isinstance(v_inst, Variable):
-            return
-        if v_result.concretetype != graph.getreturnvar().concretetype:
-            return
-        if v_result.concretetype == lltype.Void:
-            return
-        argnum = i = 0
-        while block.inputargs[i] is not v_inst:
-            if block.inputargs[i].concretetype != lltype.Void:
-                argnum += 1
-            i += 1
-        PSTRUCT = v_inst.concretetype
-        v_check = v_result
-        fastcase = True
-        for op in operations:
-            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
-                    and v_check is op.args[0]):
-                v_check = op.result
-            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
-                v_check = op.result
-                fastcase = not fastcase
-            elif (op.opname in ('int_eq', 'int_ne')
-                    and v_check is op.args[0]
-                    and isinstance(op.args[1], Constant)
-                    and op.args[1].value == 0):
-                v_check = op.result
-                if op.opname == 'int_eq':
-                    fastcase = not fastcase
-            else:
-                return
-        if v_check.concretetype is not lltype.Bool:
-            return
-        if block.exitswitch is not v_check:
-            return
-
-        links = [link for link in block.exits if link.exitcase == fastcase]
-        if len(links) != 1:
-            return
-        [link] = links
-        if link.args != [v_result]:
-            return
-        if not link.target.is_final_block():
-            return
-
-        if c_fieldname is not None:
-            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
-        else:
-            fielddescr = None
-        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,8 +117,7 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True,
-                call_shortcut=None):
+                can_collect=True):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -136,8 +135,7 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect,
-               call_shortcut)
+               can_collect)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -192,7 +190,6 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
-        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -278,8 +275,7 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True,
-                                 call_shortcut=None):
+                                 can_collect=True):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -368,8 +364,7 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect,
-                      call_shortcut)
+                      can_collect)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -392,24 +387,6 @@
 
 # ____________________________________________________________
 
-
-class CallShortcut(object):
-    def __init__(self, argnum, fielddescr):
-        self.argnum = argnum
-        self.fielddescr = fielddescr
-
-    def __eq__(self, other):
-        return (isinstance(other, CallShortcut) and
-                self.argnum == other.argnum and
-                self.fielddescr == other.fielddescr)
-    def __ne__(self, other):
-        return not (self == other)
-    def __hash__(self):
-        return hash((self.argnum, self.fielddescr))
-
-# ____________________________________________________________
-
-
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -789,12 +789,20 @@
                                                 arrayfielddescr,
                                                 arraydescr)
             return []
+        # check for the string or unicode hash field
+        STRUCT = v_inst.concretetype.TO
+        if STRUCT == rstr.STR:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('strhash', [v_inst], op.result)
+        elif STRUCT == rstr.UNICODE:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('unicodehash', [v_inst], op.result)
         # check for _immutable_fields_ hints
-        immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+        immut = STRUCT._immutable_field(c_fieldname.value)
         need_live = False
         if immut:
             if (self.callcontrol is not None and
-                self.callcontrol.could_be_green_field(v_inst.concretetype.TO,
+                self.callcontrol.could_be_green_field(STRUCT,
                                                       c_fieldname.value)):
                 pure = '_greenfield'
                 need_live = True
@@ -802,10 +810,9 @@
                 pure = '_pure'
         else:
             pure = ''
-        self.check_field_access(v_inst.concretetype.TO)
-        argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
-        descr = self.cpu.fielddescrof(v_inst.concretetype.TO,
-                                      c_fieldname.value)
+        self.check_field_access(STRUCT)
+        argname = getattr(STRUCT, '_gckind', 'gc')
+        descr = self.cpu.fielddescrof(STRUCT, c_fieldname.value)
         kind = getkind(RESULT)[0]
         if argname != 'gc':
             assert argname == 'raw'
@@ -822,7 +829,7 @@
         if immut in (IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY):
             op1.opname += "_pure"
             descr1 = self.cpu.fielddescrof(
-                v_inst.concretetype.TO,
+                STRUCT,
                 quasiimmut.get_mutate_field_name(c_fieldname.value))
             return [SpaceOperation('-live-', [], None),
                    SpaceOperation('record_quasiimmut_field',
@@ -1569,7 +1576,7 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
+    def _rewrite_op_cond_call(self, op, rewritten_opname):
         have_floats = False
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
@@ -1580,13 +1587,18 @@
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
-        op1 = self.rewrite_call(op, 'conditional_call',
+        op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
                                 calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_conditional_call(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call')
+    def rewrite_op_jit_conditional_call_value(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call_value')
+
     def handle_jit_marker__jit_merge_point(self, op, jitdriver):
         assert self.portal_jd is not None, (
             "'jit_merge_point' in non-portal graph!")
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+from rpython.jit.codewriter.effectinfo import EffectInfo
 
 
 class FakePolicy:
@@ -368,121 +368,3 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
-
-def test_find_call_shortcut():
-    class FakeCPU:
-        def fielddescrof(self, TYPE, fieldname):
-            if isinstance(TYPE, lltype.GcStruct):
-                if fieldname == 'inst_foobar':
-                    return 'foobardescr'
-                if fieldname == 'inst_fooref':
-                    return 'foorefdescr'
-            if TYPE == RAW and fieldname == 'x':
-                return 'xdescr'
-            assert False, (TYPE, fieldname)
-    cc = CallControl(FakeCPU())
-
-    class B(object):
-        foobar = 0
-        fooref = None
-
-    def f1(a, b, c):
-        if b.foobar:
-            return b.foobar
-        b.foobar = a + c
-        return b.foobar
-
-    def f2(x, y, z, b):
-        r = b.fooref
-        if r is not None:
-            return r
-        r = b.fooref = B()
-        return r
-
-    class Space(object):
-        def _freeze_(self):
-            return True
-    space = Space()
-
-    def f3(space, b):
-        r = b.foobar
-        if not r:
-            r = b.foobar = 123
-        return r
-
-    def f4(raw):
-        r = raw.x
-        if r != 0:
-            return r
-        raw.x = 123
-        return 123
-    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
-
-    def f5(b):
-        r = b.foobar
-        if r == 0:
-            r = b.foobar = 123
-        return r
-
-    def f6(b):
-        if b is not None:
-            return b
-        return B()
-
-    def f7(c, a):
-        if a:
-            return a
-        return 123
-
-    def b_or_none(c):
-        if c > 15:
-            return B()
-        return None
-
-    def f(a, c):
-        b = B()
-        f1(a, b, c)
-        f2(a, c, a, b)
-        f3(space, b)
-        r = lltype.malloc(RAW, flavor='raw')
-        f4(r)
-        f5(b)
-        f6(b_or_none(c))
-        f7(c, a)
-
-    rtyper = support.annotate(f, [10, 20])
-    f1_graph = rtyper.annotator.translator._graphof(f1)
-    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
-    f2_graph = rtyper.annotator.translator._graphof(f2)
-    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
-    f3_graph = rtyper.annotator.translator._graphof(f3)
-    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
-    f4_graph = rtyper.annotator.translator._graphof(f4)
-    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
-    f5_graph = rtyper.annotator.translator._graphof(f5)
-    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
-    f6_graph = rtyper.annotator.translator._graphof(f6)
-    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
-    f7_graph = rtyper.annotator.translator._graphof(f7)
-    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
-
-def test_cant_find_call_shortcut():
-    from rpython.jit.backend.llgraph.runner import LLGraphCPU
-
-    @jit.dont_look_inside
-    @jit.call_shortcut
-    def f1(n):
-        return n + 17   # no call shortcut found
-
-    def f(n):
-        return f1(n)
-
-    rtyper = support.annotate(f, [1])
-    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
-    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
-    res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
-    call_op = f_graph.startblock.operations[0]
-    assert call_op.opname == 'direct_call'
-    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
-    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1199,13 +1199,27 @@
     def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr):
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
-    # conditional calls - note that they cannot return stuff
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
+        # conditional calls - condition is a flag, and they cannot return stuff
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
+    @arguments("cpu", "i", "i", "I", "R", "d", returns="i")
+    def bhimpl_conditional_call_value_ir_i(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if value == 0:
+            value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
+        return value
+
+    @arguments("cpu", "r", "i", "I", "R", "d", returns="r")
+    def bhimpl_conditional_call_value_ir_r(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if not value:
+            value = cpu.bh_call_r(func, args_i, args_r, None, calldescr)
+        return value
+
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
@@ -1493,6 +1507,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_strhash(cpu, string):
+        return cpu.bh_strhash(string)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1509,6 +1526,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_unicodehash(cpu, unicode):
+        return cpu.bh_unicodehash(unicode)
 
     @arguments("i", "i")
     def bhimpl_rvmprof_code(leaving, unique_id):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,6 +271,7 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
+            OpHelpers.is_cond_call_value(opnum) or
             opnum == rop.COND_CALL):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,5 +1,5 @@
 
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
@@ -73,6 +73,9 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
+    def getstrhash(self, op, mode):
+        return None
+
     def copy_fields_to_const(self, constinfo, optheap):
         pass
 
@@ -790,6 +793,20 @@
                 return None
             return ConstInt(len(s))
 
+    def getstrhash(self, op, mode):
+        from rpython.jit.metainterp.optimizeopt import vstring
+
+        if mode is vstring.mode_string:
+            s = self._unpack_str(vstring.mode_string)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+        else:
+            s = self._unpack_str(vstring.mode_unicode)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -261,9 +261,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
-    def _can_optimize_call_pure(self, op):
+    def _can_optimize_call_pure(self, op, start_index=0):
         arg_consts = []
-        for i in range(op.numargs()):
+        for i in range(start_index, op.numargs()):
             arg = op.getarg(i)
             const = self.optimizer.get_constant_box(arg)
             if const is None:
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -151,13 +151,13 @@
             self._pure_operations[opnum] = recentops = RecentPureOps()
         return recentops
 
-    def optimize_CALL_PURE_I(self, op):
+    def optimize_call_pure(self, op, start_index=0):
         # Step 1: check if all arguments are constant
-        for arg in op.getarglist():
-            self.optimizer.force_box(arg)
+        for i in range(start_index, op.numargs()):
+            self.optimizer.force_box(op.getarg(i))
             # XXX hack to ensure that virtuals that are
             #     constant are presented that way
-        result = self._can_optimize_call_pure(op)
+        result = self._can_optimize_call_pure(op, start_index=start_index)
         if result is not None:
             # this removes a CALL_PURE with all constant arguments.
             self.make_constant(op, result)
@@ -168,32 +168,46 @@
         # CALL_PURE.
         for pos in self.call_pure_positions:
             old_op = self.optimizer._newoperations[pos]
-            if self.optimize_call_pure(op, old_op):
+            if self.optimize_call_pure_old(op, old_op, start_index):
                 return
         if self.extra_call_pure:
             for i, old_op in enumerate(self.extra_call_pure):
-                if self.optimize_call_pure(op, old_op):
+                if self.optimize_call_pure_old(op, old_op, start_index):
                     if isinstance(old_op, PreambleOp):
                         old_op = self.optimizer.force_op_from_preamble(old_op)
                         self.extra_call_pure[i] = old_op
                     return
 
-        # replace CALL_PURE with just CALL
-        opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = self.optimizer.replace_op_with(op, opnum)
+        # replace CALL_PURE with just CALL (but keep COND_CALL_VALUE)
+        if start_index == 0:
+            opnum = OpHelpers.call_for_descr(op.getdescr())
+            newop = self.optimizer.replace_op_with(op, opnum)
+        else:
+            newop = op
         return self.emit_result(CallPureOptimizationResult(self, newop))
 
+    def optimize_CALL_PURE_I(self, op):
+        return self.optimize_call_pure(op)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
-    def optimize_call_pure(self, op, old_op):
-        if (op.numargs() != old_op.numargs() or
-            op.getdescr() is not old_op.getdescr()):
+    def optimize_COND_CALL_VALUE_I(self, op):
+        return self.optimize_call_pure(op, start_index=1)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
+    def optimize_call_pure_old(self, op, old_op, start_index):
+        if op.getdescr() is not old_op.getdescr():
             return False
-        for i, box in enumerate(old_op.getarglist()):
-            if not self.get_box_replacement(op.getarg(i)).same_box(box):
+        # this will match a call_pure and a cond_call_value with
+        # the same function and arguments
+        j = start_index
+        old_start_index = OpHelpers.is_cond_call_value(old_op.opnum)
+        for i in range(old_start_index, old_op.numargs()):
+            box = old_op.getarg(i)
+            if not self.get_box_replacement(op.getarg(j)).same_box(box):
                 break
+            j += 1
         else:
             # all identical
             # this removes a CALL_PURE that has the same (non-constant)
@@ -250,10 +264,17 @@
             # don't move call_pure_with_exception in the short preamble...
             # issue #2015
 
+            # Also, don't move cond_call_value in the short preamble.
+            # The issue there is that it's usually pointless to try to
+            # because the 'value' argument is typically not a loop
+            # invariant, and would really need to be in order to end up
+            # in the short preamble.  Maybe the code works anyway in the
+            # other rare case, but better safe than sorry and don't try.
             effectinfo = op.getdescr().get_extra_info()
             if not effectinfo.check_can_raise(ignore_memoryerror=True):
                 assert rop.is_call(op.opnum)
-                sb.add_pure_op(op)
+                if not OpHelpers.is_cond_call_value(op.opnum):
+                    sb.add_pure_op(op)
 
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -596,6 +596,19 @@
             op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         return self.emit(op)
 
+    def optimize_COND_CALL_VALUE_I(self, op):
+        # look if we know the nullness of the first argument
+        info = self.getnullness(op.getarg(0))
+        if info == INFO_NONNULL:
+            self.make_equal_to(op, op.getarg(0))
+            self.last_emitted_operation = REMOVED
+            return
+        if info == INFO_NULL:
+            opnum = OpHelpers.call_pure_for_type(op.type)
+            op = self.replace_op_with(op, opnum, args=op.getarglist()[1:])
+        return self.emit(op)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
     def _optimize_nullness(self, op, box, expect_nonnull):
         info = self.getnullness(box)
         if info == INFO_NONNULL:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, TreeLoop
+from rpython.jit.metainterp.history import ConstPtr
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import build_opt_chain
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
@@ -8674,6 +8675,75 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_with_a_constant_i(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(0, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i2 = call_i(123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_with_a_constant_i2(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(12, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        escape_n(12)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r1(self):
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        jump(p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_cond_call_r2(self):
+        ops = """
+        [p1]
+        guard_nonnull(p1) []
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        guard_nonnull(p1) []
+        p3 = escape_r(p1)
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r3(self):
+        arg_consts = [ConstInt(i) for i in (123, 4, 5, 6)]
+        call_pure_results = {tuple(arg_consts): ConstPtr(self.myptr)}
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, 4, 5, 6, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        p3 = escape_r(ConstPtr(myptr))
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -438,6 +438,10 @@
                     oopspecindex=EffectInfo.OS_INT_PY_MOD)
     int_py_mod_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
 
+    FUNC = lltype.FuncType([], llmemory.GCREF)
+    ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE)
+    plain_r_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
+
     namespace = locals()
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -537,6 +537,20 @@
                 return
         return self.emit(op)
 
+    def optimize_STRHASH(self, op):
+        return self._optimize_STRHASH(op, mode_string)
+    def optimize_UNICODEHASH(self, op):
+        return self._optimize_STRHASH(op, mode_unicode)
+
+    def _optimize_STRHASH(self, op, mode):
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            lgtop = opinfo.getstrhash(op, mode)
+            if lgtop is not None:
+                self.make_equal_to(op, lgtop)
+                return
+        return self.emit(op)
+
     def optimize_COPYSTRCONTENT(self, op):
         return self._optimize_COPYSTRCONTENT(op, mode_string)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1059,8 +1059,21 @@
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
+        if isinstance(condbox, ConstInt) and condbox.value == 0:
+            return   # so that the heapcache can keep argboxes virtual
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
+    @arguments("box", "box", "boxes2", "descr", "orgpc")
+    def _opimpl_conditional_call_value(self, valuebox, funcbox, argboxes,
+                                       calldescr, pc):
+        if isinstance(valuebox, Const) and valuebox.nonnull():
+            return valuebox
+        return self.do_conditional_call(valuebox, funcbox, argboxes,
+                                        calldescr, pc, is_value=True)
+
+    opimpl_conditional_call_value_ir_i = _opimpl_conditional_call_value
+    opimpl_conditional_call_value_ir_r = _opimpl_conditional_call_value
+
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
@@ -1150,6 +1163,20 @@
         self.execute(rop.UNICODESETITEM, unicodebox, indexbox, newcharbox)
 
     @arguments("box")
+    def opimpl_strhash(self, strbox):
+        if isinstance(strbox, ConstPtr):
+            h = self.metainterp.cpu.bh_strhash(strbox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.STRHASH, strbox)
+
+    @arguments("box")
+    def opimpl_unicodehash(self, unicodebox):
+        if isinstance(unicodebox, ConstPtr):
+            h = self.metainterp.cpu.bh_unicodehash(unicodebox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.UNICODEHASH, unicodebox)
+
+    @arguments("box")
     def opimpl_newstr(self, lengthbox):
         return self.execute(rop.NEWSTR, lengthbox)
 
@@ -1538,7 +1565,7 @@
                                                             descr=descr)
         if pure and not self.metainterp.last_exc_value and op:
             op = self.metainterp.record_result_of_call_pure(op, argboxes, descr,
-                patch_pos)
+                patch_pos, opnum)
             exc = exc and not isinstance(op, Const)
         if exc:
             if op is not None:
@@ -1712,16 +1739,31 @@
             else:
                 assert False
 
-    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
-        if isinstance(condbox, ConstInt) and condbox.value == 0:
-            return   # so that the heapcache can keep argboxes virtual
+    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc,
+                            is_value=False):
         allboxes = self._build_allboxes(funcbox, argboxes, descr)
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop.COND_CALL, [condbox] + allboxes, descr,
-                                    exc, pure)
+        allboxes = [condbox] + allboxes
+        # COND_CALL cannot be pure (=elidable): it has no result.
+        # On the other hand, COND_CALL_VALUE is always calling a pure
+        # function.
+        if not is_value:
+            return self.execute_varargs(rop.COND_CALL, allboxes, descr,
+                                        exc, pure=False)
+        else:
+            opnum = OpHelpers.cond_call_value_for_descr(descr)
+            # work around the fact that execute_varargs() wants a
+            # constant for first argument
+            if opnum == rop.COND_CALL_VALUE_I:
+                return self.execute_varargs(rop.COND_CALL_VALUE_I, allboxes,
+                                            descr, exc, pure=True)
+            elif opnum == rop.COND_CALL_VALUE_R:
+                return self.execute_varargs(rop.COND_CALL_VALUE_R, allboxes,
+                                            descr, exc, pure=True)
+            else:
+                raise AssertionError
 
     def _do_jit_force_virtual(self, allboxes, descr, pc):
         assert len(allboxes) == 2
@@ -3061,11 +3103,16 @@
         debug_stop("jit-abort-longest-function")
         return max_jdsd, max_key
 
-    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos):
+    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos, opnum):
         """ Patch a CALL into a CALL_PURE.
         """
         resbox_as_const = executor.constant_from_op(op)
-        for argbox in argboxes:
+        is_cond_value = OpHelpers.is_cond_call_value(opnum)
+        if is_cond_value:
+            normargboxes = argboxes[1:]    # ingore the 'value' arg
+        else:
+            normargboxes = argboxes
+        for argbox in normargboxes:
             if not isinstance(argbox, Const):
                 break
         else:
@@ -3075,8 +3122,10 @@
             return resbox_as_const
         # not all constants (so far): turn CALL into CALL_PURE, which might
         # be either removed later by optimizeopt or turned back into CALL.
-        arg_consts = [executor.constant_from_op(a) for a in argboxes]
+        arg_consts = [executor.constant_from_op(a) for a in normargboxes]
         self.call_pure_results[arg_consts] = resbox_as_const
+        if is_cond_value:
+            return op       # but COND_CALL_VALUE remains
         opnum = OpHelpers.call_pure_for_descr(descr)
         self.history.cut(patch_pos)
         newop = self.history.record_nospec(opnum, argboxes, descr)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -1097,6 +1097,8 @@
     '_MALLOC_LAST',
     'FORCE_TOKEN/0/r',    # historical name; nowadays, returns the jitframe
     'VIRTUAL_REF/2/r',    # removed before it's passed to the backend
+    'STRHASH/1/i',        # only reading the .hash field, might be zero so far
+    'UNICODEHASH/1/i',    #     (unless applied on consts, where .hash is forced)
     # this one has no *visible* side effect, since the virtualizable
     # must be forced, however we need to execute it anyway
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
@@ -1151,7 +1153,7 @@
     '_CALL_FIRST',
     'CALL/*d/rfin',
     'COND_CALL/*d/n',   # a conditional call, with first argument as a condition
-    'COND_CALL_VALUE/*d/ri',  # same but returns a result; emitted by rewrite
+    'COND_CALL_VALUE/*d/ri',  # "return a0 or a1(a2, ..)", a1 elidable
     'CALL_ASSEMBLER/*d/rfin',  # call already compiled assembler
     'CALL_MAY_FORCE/*d/rfin',
     'CALL_LOOPINVARIANT/*d/rfin',
@@ -1274,6 +1276,15 @@
         return rop.CALL_LOOPINVARIANT_N
 
     @staticmethod
+    def cond_call_value_for_descr(descr):
+        tp = descr.get_normalized_result_type()
+        if tp == 'i':
+            return rop.COND_CALL_VALUE_I
+        elif tp == 'r':
+            return rop.COND_CALL_VALUE_R
+        assert False, tp
+
+    @staticmethod
     def getfield_pure_for_descr(descr):
         if descr.is_pointer_field():
             return rop.GETFIELD_GC_PURE_R
@@ -1326,6 +1337,16 @@
         return rop.CALL_N
 
     @staticmethod
+    def call_pure_for_type(tp):
+        if tp == 'i':
+            return rop.CALL_PURE_I
+        elif tp == 'r':
+            return rop.CALL_PURE_R
+        elif tp == 'f':
+            return rop.CALL_PURE_F
+        return rop.CALL_PURE_N
+
+    @staticmethod
     def is_guard(opnum):
         return rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST
 
@@ -1447,6 +1468,11 @@
                 opnum == rop.CALL_RELEASE_GIL_N)
 
     @staticmethod
+    def is_cond_call_value(opnum):
+        return (opnum == rop.COND_CALL_VALUE_I or
+                opnum == rop.COND_CALL_VALUE_R)
+
+    @staticmethod
     def is_ovf(opnum):
         return rop._OVF_FIRST <= opnum <= rop._OVF_LAST
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3775,9 +3775,8 @@
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
-        self.check_resops({'int_gt': 2, 'getfield_gc_i': 1, 'int_eq': 1,
-                           'guard_true': 2, 'int_sub': 2, 'jump': 1,
-                           'guard_false': 1})
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
 
     def test_virtual_after_bridge(self):
         myjitdriver = JitDriver(greens = [], reds = ["n"])
@@ -4575,3 +4574,14 @@
 
         self.meta_interp(g, [5, 5, 5])
         self.check_resops(guard_true=10)   # 5 unrolled, plus 5 unrelated
+
+    def test_conditional_call_value(self):
+        from rpython.rlib.jit import conditional_call_elidable
+        def g(j):
+            return j + 5
+        def f(i, j):
+            return conditional_call_elidable(i, g, j)
+        res = self.interp_operations(f, [-42, 200])
+        assert res == -42
+        res = self.interp_operations(f, [0, 200])
+        assert res == 205
diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -1,8 +1,8 @@
 
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, noConst
 from rpython.rlib import jit
 
-class TestCall(LLJitMixin):
+class CallTest(object):
     def test_indirect_call(self):
         @jit.dont_look_inside
         def f1(x):
@@ -52,3 +52,230 @@
 
         assert self.meta_interp(main, [10]) == 42
         self.check_resops(guard_no_exception=0)
+
+    def test_cond_call_i(self):
+        def f(n):
+            return n * 200
+
+        def main(n, m):
+            return jit.conditional_call_elidable(n, f, m)
+
+        assert self.interp_operations(main, [0, 10]) == 2000
+        assert self.interp_operations(main, [15, 42]) == 15
+
+    def test_cond_call_r(self):
+        def f(n):
+            return [n]
+
+        def main(n):
+            if n == 10:
+                l = []
+            else:
+                l = None
+            l = jit.conditional_call_elidable(l, f, n)
+            return len(l)
+
+        assert main(10) == 0
+        assert main(5) == 1
+        assert self.interp_operations(main, [10]) == 0
+        assert self.interp_operations(main, [5]) == 1
+
+    def test_cond_call_constant_in_pyjitpl(self):
+        def f(a, b):
+            return a + b
+        def main(n):
+            # this is completely constant-folded because the arguments
+            # to f() are constants.
+            return jit.conditional_call_elidable(n, f, 40, 2)
+
+        assert main(12) == 12
+        assert main(0) == 42
+        assert self.interp_operations(main, [12]) == 12
+        self.check_operations_history({'finish': 1})   # empty history
+        assert self.interp_operations(main, [0]) == 42
+        self.check_operations_history({'finish': 1})   # empty history
+
+    def test_cond_call_constant_in_optimizer(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return x - 3
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                m1 = noConst(m)
+                n -= jit.conditional_call_elidable(p, externfn, m1)
+            return n
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        # the COND_CALL_VALUE is constant-folded away by optimizeopt.py
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
+
+    def test_cond_call_constant_in_optimizer_1(self):
+        # same as test_cond_call_constant_in_optimizer, but the 'value'
+        # argument changes
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return x - 3
+        class V:
+            def __init__(self, value):
+                self.value = value
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                m1 = noConst(m)
+                n -= jit.conditional_call_elidable(p, externfn, m1)
+            return n
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        # the COND_CALL_VALUE is constant-folded away by optimizeopt.py
+        self.check_resops({'int_sub': 2, 'int_gt': 2, 'guard_true': 2,
+                           'jump': 1})
+
+    def test_cond_call_constant_in_optimizer_2(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return 2
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > -1
+                assert p < 1
+                n -= jit.conditional_call_elidable(p, externfn, n)
+            return n
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        # optimizer: the COND_CALL_VALUE is turned into a regular
+        # CALL_PURE, which itself becomes a CALL
+        self.check_resops(call_pure_i=0, cond_call_value_i=0, call_i=2,
+                          int_sub=2)
+
+    def test_cond_call_constant_in_optimizer_3(self):
+        myjitdriver = jit.JitDriver(greens = ['m'], reds = ['n', 'p'])
+        def externfn(x):
+            return 1
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                assert p > -1
+                assert p < 1
+                n0 = n
+                n -= jit.conditional_call_elidable(p, externfn, n0)
+                n -= jit.conditional_call_elidable(p, externfn, n0)
+            return n
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        # same as test_cond_call_constant_in_optimizer_2, but the two
+        # intermediate CALL_PUREs are replaced with only one, because
+        # they are called with the same arguments
+        self.check_resops(call_pure_i=0, cond_call_value_i=0, call_i=2,
+                          int_sub=4)
+
+    def test_cond_call_constant_in_optimizer_4(self):
+        class X:
+            def __init__(self, value):
+                self.value = value
+                self.triple = 0
+            def _compute_triple(self):
+                self.triple = self.value * 3
+                return self.triple
+            def get_triple(self):
+                return jit.conditional_call_elidable(self.triple,
+                                                  X._compute_triple, self)
+
+        myjitdriver = jit.JitDriver(greens = [], reds = 'auto')
+        def main(n):
+            total = 0
+            while n > 1:
+                myjitdriver.jit_merge_point()
+                x = X(n)
+                total += x.get_triple() + x.get_triple() + x.get_triple()
+                n -= 10
+            return total
+
+        res = self.meta_interp(main, [100])
+        assert res == main(100)
+        # remaining: only the first call to get_triple(), as a call_i
+        # because we know that x.triple == 0 here.  The remaining calls
+        # are removed because equal to the first one.
+        self.check_resops(call_i=2, cond_call_value_i=0)
+
+    def test_cond_call_multiple_in_optimizer_1(self):
+        # test called several times with the same arguments, but
+        # the condition is not available to the short preamble.
+        # This means that the second cond_call_value after unrolling
+        # can't be removed.
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2000      # never actually called
+        @jit.dont_look_inside
+        def randomish(p):
+            return p + 1
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                n -= jit.conditional_call_elidable(randomish(p), externfn, m)
+            return n
+        assert f(21, 5, 1) == -1
+        res = self.meta_interp(f, [21, 5, 1])
+        assert res == -1
+        self.check_resops(call_pure_i=0, cond_call_value_i=2,
+                          call_i=2,    # randomish()
+                          int_sub=2)
+
+    def test_cond_call_multiple_in_optimizer_2(self):
+        # test called several times with the same arguments.  Ideally
+        # we would like them to be consolidated into one call even if
+        # the 'value' are different but available from the short
+        # preamble.  We don't do it so far---it's a mess, because the
+        # short preamble is supposed to depend only on loop-invariant
+        # things, and 'value' is (most of the time) not loop-invariant.
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2      # called only the first time
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                p = jit.conditional_call_elidable(p, externfn, m)
+                n -= p
+            return n
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+        self.check_resops(call_pure_i=0,
+                          cond_call_value_i=2,   # ideally 1, but see above
+                          int_sub=2)
+
+    def test_cond_call_in_blackhole(self):
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x):
+            return 2
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                if n > 6:    # will fail and finish in the blackhole
+                    pass
+                if jit.we_are_jitted():   # manually inline here
+                    p = jit._jit_conditional_call_value(p, externfn, m)
+                else:
+                    p = jit.conditional_call_elidable(p, externfn, m)
+                n -= p
+            return n
+        assert f(21, 5, 0) == -1
+        res = self.meta_interp(f, [21, 5, 0])
+        assert res == -1
+
+
+class TestCall(LLJitMixin, CallTest):
+    pass
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -194,7 +194,8 @@
                            'guard_true': 4, 'jump': 1,
                            'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
-                           'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
+                           'call_i': 4, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
+                           'cond_call_value_i': 2, 'strhash': 4,
                            'guard_no_exception': 8, 'new': 2,
                            'guard_nonnull': 2})
 
@@ -273,7 +274,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call_i=4, call_n=1)
+        self.check_simple_loop(call_i=3, cond_call_value_i=1, call_n=1)
 
     def test_dict_array_write_invalidates_caches(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -295,7 +296,7 @@
         exp = f(10)
         res = self.meta_interp(f, [10])
         assert res == exp
-        self.check_simple_loop(call_i=5, call_n=2)
+        self.check_simple_loop(call_i=4, cond_call_value_i=1, call_n=2)
 
     def test_dict_double_lookup_2(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -314,7 +315,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call_i=2, call_n=1)
+        self.check_simple_loop(call_i=1, cond_call_value_i=1, call_n=1)
 
     def test_dict_eq_can_release_gil(self):
         from rpython.rtyper.lltypesystem import lltype, rffi
diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -951,3 +951,10 @@
         self.meta_interp(f, [222, 3333])
         self.check_simple_loop({'guard_true': 1, 'int_add': 1,
                                 'int_lt': 1, 'jump': 1})
+
+    def test_string_hashing(self):
+        def f(i):
+            s = str(i)
+            d = {s: s + s}
+            return len(d[s])
+        assert self.interp_operations(f, [222]) == 6
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -257,27 +257,6 @@
     func.oopspec = "jit.not_in_trace()"   # note that 'func' may take arguments
     return func
 
-def call_shortcut(func):
-    """A decorator to ensure that a function has a fast-path.
-    DOES NOT RELIABLY WORK ON METHODS, USE ONLY ON FUNCTIONS!
-
-    Only useful on functions that the JIT doesn't normally look inside.
-    It still replaces residual calls to that function with inline code
-    that checks for a fast path, and only does the call if not.  For
-    now, graphs made by the following kinds of functions are detected:
-
-           def func(x, y, z):         def func(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r is None:
-               ...                             ...
-                                           return r
-
-    Fast-path detection is always on, but this decorator makes the
-    codewriter complain if it cannot find the promized fast-path.
-    """
-    func._call_shortcut_ = True
-    return func
-
 
 @oopspec("jit.isconstant(value)")
 @specialize.call_location()
@@ -1194,10 +1173,19 @@
         return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
-    pass
+    pass           # special-cased below
 
 @specialize.call_location()
 def conditional_call(condition, function, *args):
+    """Does the same as:
+
+         if condition:
+             function(*args)
+
+    but is better for the JIT, in case the condition is often false
+    but could be true occasionally.  It allows the JIT to always produce
+    bridge-free code.  The function is never looked inside.
+    """
     if we_are_jitted():
         _jit_conditional_call(condition, function, *args)
     else:
@@ -1205,21 +1193,76 @@
             function(*args)
 conditional_call._always_inline_ = True
 
+def _jit_conditional_call_value(value, function, *args):
+    return value    # special-cased below
+
+ at specialize.call_location()
+def conditional_call_elidable(value, function, *args):
+    """Does the same as:
+
+        if value == <0 or None>:
+            value = function(*args)
+        return value
+
+    For the JIT.  Allows one branch which doesn't create a bridge,
+    typically used for caching.  The value and the function's return
+    type must match and cannot be a float: they must be either regular
+    'int', or something that turns into a pointer.
+
+    Even if the function is not marked @elidable, it is still treated
+    mostly like one.  The only difference is that (in heapcache.py)
+    we don't assume this function won't change anything observable.
+    This is useful for caches, as you can write:
+
+        def _compute_and_cache(...):
+            self.cache = ...compute...
+            return self.cache
+
+        x = jit.conditional_call_elidable(self.cache, _compute_and_cache, ...)
+
+    """
+    if we_are_jitted():
+        return _jit_conditional_call_value(value, function, *args)
+    else:
+        if isinstance(value, int):
+            if value == 0:
+                value = function(*args)
+                assert isinstance(value, int)
+        else:
+            if value is None:
+                value = function(*args)
+                assert not isinstance(value, int)
+        return value
+conditional_call_elidable._always_inline_ = True
+
 class ConditionalCallEntry(ExtRegistryEntry):
-    _about_ = _jit_conditional_call
+    _about_ = _jit_conditional_call, _jit_conditional_call_value
 
     def compute_result_annotation(self, *args_s):
-        self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
-                                         args_s[1], args_s[2:])
+        s_res = self.bookkeeper.emulate_pbc_call(self.bookkeeper.position_key,
+                                                 args_s[1], args_s[2:])
+        if self.instance == _jit_conditional_call_value:
+            from rpython.annotator import model as annmodel
+            return annmodel.unionof(s_res, args_s[0])
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
 
-        args_v = hop.inputargs(lltype.Bool, lltype.Void, *hop.args_r[2:])
+        if self.instance == _jit_conditional_call:
+            opname = 'jit_conditional_call'
+            COND = lltype.Bool
+            resulttype = None
+        elif self.instance == _jit_conditional_call_value:
+            opname = 'jit_conditional_call_value'
+            COND = hop.r_result
+            resulttype = hop.r_result.lowleveltype
+        else:
+            assert False
+        args_v = hop.inputargs(COND, lltype.Void, *hop.args_r[2:])
         args_v[1] = hop.args_r[1].get_concrete_llfn(hop.args_s[1],
                                                     hop.args_s[2:], hop.spaceop)
         hop.exception_is_here()
-        return hop.genop('jit_conditional_call', args_v)
+        return hop.genop(opname, args_v, resulttype=resulttype)
 
 def enter_portal_frame(unique_id):
     """call this when starting to interpret a function. calling this is not
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -28,6 +28,8 @@
 class FakeWeakCodeObjectList(object):
     def add_handle(self, handle):

From pypy.commits at gmail.com  Thu Nov 24 07:37:03 2016
From: pypy.commits at gmail.com (mattip)
Date: Thu, 24 Nov 2016 04:37:03 -0800 (PST)
Subject: [pypy-commit] pypy better-PyDict_Next: pass tests,
 except for test_typedict
Message-ID: <5836deef.0f341c0a.13702.bcd7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: better-PyDict_Next
Changeset: r88635:f2c9d8b8ab2a
Date: 2016-11-24 14:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f2c9d8b8ab2a/

Log:	pass tests, except for test_typedict

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -610,7 +610,7 @@
         GLOBALS['%s#%s' % (cpyname, pypy_decl)] = ('PyTypeObject*', pypyexpr)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
-                      PyDictObject PyClassObject'''.split():
+                      PyClassObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py
--- a/pypy/module/cpyext/bytearrayobject.py
+++ b/pypy/module/cpyext/bytearrayobject.py
@@ -7,7 +7,7 @@
     PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
-    PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
+    PyObject, PyObjectP, Py_DecRef, make_ref, from_ref,
     make_typedescr, get_typedescr, Py_IncRef)
 # Type PyByteArrayObject represents a mutable array of bytes.
 # The Python API is that of a sequence;
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -1,12 +1,51 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
-    Py_ssize_tP, CONST_STRING)
-from pypy.module.cpyext.pyobject import PyObject, PyObjectP, as_pyobj
+    Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
+    bootstrap_function)
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+        make_typedescr, track_reference, create_ref, from_ref, Py_DecRef,
+        Py_IncRef)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import specialize
 
+PyDictObjectStruct = lltype.ForwardReference()
+PyDictObject = lltype.Ptr(PyDictObjectStruct)
+PyDictObjectFields = PyObjectFields + \
+    (("ob_keys", PyObject),)
+cpython_struct("PyDictObject", PyDictObjectFields, PyDictObjectStruct)
+
+ at bootstrap_function
+def init_dictobject(space):
+    "Type description of PyDictObject"
+    make_typedescr(space.w_dict.layout.typedef,
+                   basestruct=PyDictObject.TO,
+                   attach=dict_attach,
+                   dealloc=dict_dealloc,
+                   realize=dict_realize)
+
+def dict_attach(space, py_obj, w_obj):
+    """
+    Fills a newly allocated PyDictObject with the given dict object.
+    """
+    py_dict = rffi.cast(PyDictObject, py_obj)
+    py_dict.c_ob_keys = lltype.nullptr(PyObject.TO)
+
+def dict_realize(space, py_obj):
+    """
+    Creates the dict in the interpreter
+    """
+    w_obj = space.newdict()
+    track_reference(space, py_obj, w_obj)
+
+ at cpython_api([PyObject], lltype.Void, header=None)
+def dict_dealloc(space, py_obj):
+    py_dict = rffi.cast(PyDictObject, py_obj)
+    Py_DecRef(space, py_dict.c_ob_keys)
+    from pypy.module.cpyext.object import _dealloc
+    _dealloc(space, py_obj)
+
 @cpython_api([], PyObject)
 def PyDict_New(space):
     return space.newdict()
@@ -181,9 +220,9 @@
     }
 
     The dictionary p should not be mutated during iteration.  It is safe
-    (since Python 2.1) to modify the values of the keys as you iterate over the
-    dictionary, but only so long as the set of keys does not change.  For
-    example:
+    (since Python 2.1) to modify the values but not the keys as you iterate
+    over the dictionary, the keys must not change.
+    For example:
 
     PyObject *key, *value;
     Py_ssize_t pos = 0;
@@ -199,34 +238,30 @@
         }
         Py_DECREF(o);
     }"""
+
     if w_dict is None:
         return 0
 
-    # XXX XXX PyDict_Next is not efficient. Storing an iterator would probably
-    # work, but we can't work out how to not leak it if iteration does
-    # not complete.  Alternatively, we could add some RPython-only
-    # dict-iterator method to move forward by N steps.
-
-    w_dict.ensure_object_strategy()     # make sure both keys and values can
-                                        # be borrwed
-    try:
-        w_iter = space.call_method(space.w_dict, "iteritems", w_dict)
-        pos = ppos[0]
-        while pos:
-            space.call_method(w_iter, "next")
-            pos -= 1
-
-        w_item = space.call_method(w_iter, "next")
-        w_key, w_value = space.fixedview(w_item, 2)
-        if pkey:
-            pkey[0]   = as_pyobj(space, w_key)
-        if pvalue:
-            pvalue[0] = as_pyobj(space, w_value)
-        ppos[0] += 1
-    except OperationError as e:
-        if not e.match(space, space.w_StopIteration):
-            raise
+    pos = ppos[0]
+    py_obj = as_pyobj(space, w_dict)
+    py_dict = rffi.cast(PyDictObject, py_obj)
+    if pos == 0:
+        # Store the current keys in the PyDictObject.
+        Py_DecRef(space, py_dict.c_ob_keys)
+        w_keys = space.call_method(space.w_dict, "keys", w_dict)
+        py_dict.c_ob_keys = create_ref(space, w_keys)
+        Py_IncRef(space, py_dict.c_ob_keys)
+    else:
+        w_keys = from_ref(space, py_dict.c_ob_keys)
+    ppos[0] += 1
+    if pos >= space.len_w(w_keys):
         return 0
+    w_key = space.listview(w_keys)[pos]
+    w_value = space.getitem(w_dict, w_key)
+    if pkey:
+        pkey[0]   = as_pyobj(space, w_key)
+    if pvalue:
+        pvalue[0] = as_pyobj(space, w_value)
     return 1
 
 @specialize.memo()
diff --git a/pypy/module/cpyext/include/dictobject.h b/pypy/module/cpyext/include/dictobject.h
--- a/pypy/module/cpyext/include/dictobject.h
+++ b/pypy/module/cpyext/include/dictobject.h
@@ -7,6 +7,10 @@
 extern "C" {
 #endif
 
+typedef struct {
+    PyObject_HEAD
+    PyObject *ob_keys; /* a private place to put keys during PyDict_Next */
+} PyDictObject;
 
 #ifdef __cplusplus
 }

From pypy.commits at gmail.com  Thu Nov 24 07:46:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 04:46:20 -0800 (PST)
Subject: [pypy-commit] pypy default: ovfcheck() is only for +, - * nowadays
Message-ID: <5836e11c.471ec20a.76ac6.fad8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88636:e1ed9c777e87
Date: 2016-11-24 13:45 +0100
http://bitbucket.org/pypy/pypy/changeset/e1ed9c777e87/

Log:	ovfcheck() is only for +, - * nowadays

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -11,7 +11,7 @@
 intmask  mask a possibly long value when running on CPython
          back to a signed int value
 ovfcheck check on CPython whether the result of a signed
-         integer operation did overflow
+         integer operation did overflow (add, sub, mul)
 ovfcheck_float_to_int
          convert to an integer or raise OverflowError
 ovfcheck_float_to_longlong
@@ -168,6 +168,7 @@
 def ovfcheck(r):
     # to be used as ovfcheck(x <op> y)
     # raise OverflowError if the operation did overflow
+    # Nowadays, only supports '+', '-' or '*' as the operation.
     assert not isinstance(r, r_uint), "unexpected ovf check on unsigned"
     assert not isinstance(r, r_longlong), "ovfcheck not supported on r_longlong"
     assert not isinstance(r, r_ulonglong), "ovfcheck not supported on r_ulonglong"

From pypy.commits at gmail.com  Thu Nov 24 10:47:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 07:47:47 -0800 (PST)
Subject: [pypy-commit] pypy default: don't switch strategies back and forth
 if we're calling PySequenceFast_ITEMS and PyList_GET_ITEM alternatively
Message-ID: <58370ba3.8f95c20a.2b130.3ae5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88637:c4f99ae69793
Date: 2016-11-24 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/c4f99ae69793/

Log:	don't switch strategies back and forth if we're calling
	PySequenceFast_ITEMS and PyList_GET_ITEM alternatively

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -62,12 +62,14 @@
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
+    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.ensure_object_strategy()  # make sure we can return a borrowed obj
-    # XXX ^^^ how does this interact with CPyListStrategy?
+    cpy_strategy = space.fromcache(CPyListStrategy)
+    if w_list.strategy is not cpy_strategy:
+        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
     w_res = w_list.getitem(index)
     return w_res     # borrowed ref
 

From pypy.commits at gmail.com  Thu Nov 24 10:47:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 07:47:49 -0800 (PST)
Subject: [pypy-commit] pypy default: passing test for exceptions raised by
 cond_call_value
Message-ID: <58370ba5.0f341c0a.13702.082f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88638:4d39cedda3a1
Date: 2016-11-24 14:58 +0100
http://bitbucket.org/pypy/pypy/changeset/4d39cedda3a1/

Log:	passing test for exceptions raised by cond_call_value

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -276,6 +276,28 @@
         res = self.meta_interp(f, [21, 5, 0])
         assert res == -1
 
+    def test_cond_call_raises(self):
+        myjitdriver = jit.JitDriver(greens = [], reds = ['n', 'p', 'm'])
+        def externfn(x, m):
+            if m == 1 or m == 1008:
+                raise ValueError
+            return x + m
+        def f(n, m, p):
+            while n > 0:
+                myjitdriver.can_enter_jit(n=n, p=p, m=m)
+                myjitdriver.jit_merge_point(n=n, p=p, m=m)
+                try:
+                    p = jit.conditional_call_elidable(p, externfn, n, m)
+                    p -= (n + m)   # => zero again
+                except ValueError:
+                    m += 1000
+                m += 1
+                n -= 2
+            return n * m
+        assert f(21, 0, 0) == -2011
+        res = self.meta_interp(f, [21, 0, 0])
+        assert res == -2011
+
 
 class TestCall(LLJitMixin, CallTest):
     pass

From pypy.commits at gmail.com  Thu Nov 24 10:47:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 07:47:50 -0800 (PST)
Subject: [pypy-commit] pypy default: extra passing test
Message-ID: <58370ba6.c4811c0a.2dd69.0694@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88639:2e8473885ae1
Date: 2016-11-24 15:07 +0100
http://bitbucket.org/pypy/pypy/changeset/2e8473885ae1/

Log:	extra passing test

diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py
--- a/rpython/jit/metainterp/test/test_call.py
+++ b/rpython/jit/metainterp/test/test_call.py
@@ -206,7 +206,39 @@
         # remaining: only the first call to get_triple(), as a call_i
         # because we know that x.triple == 0 here.  The remaining calls
         # are removed because equal to the first one.
-        self.check_resops(call_i=2, cond_call_value_i=0)
+        self.check_resops(call_i=2, cond_call_value_i=0,
+                          new_with_vtable=2)  # escapes: _compute_triple(self)
+
+    def test_cond_call_constant_in_optimizer_5(self):
+        def _compute_triple(value):
+            return value * 3
+        class X:
+            def __init__(self, value):
+                self.value = value
+                self.triple = 0
+            def get_triple(self):
+                res = jit.conditional_call_elidable(self.triple,
+                                                    _compute_triple, self.value)
+                self.triple = res
+                return res
+
+        myjitdriver = jit.JitDriver(greens = [], reds = 'auto')
+        def main(n):
+            total = 0
+            while n > 1:
+                myjitdriver.jit_merge_point()
+                x = X(n)
+                total += x.get_triple() + x.get_triple() + x.get_triple()
+                n -= 10
+            return total
+
+        res = self.meta_interp(main, [100])
+        assert res == main(100)
+        # remaining: only the first call to get_triple(), as a call_i
+        # because we know that x.triple == 0 here.  The remaining calls
+        # are removed because equal to the first one.
+        self.check_resops(call_i=2, cond_call_value_i=0,
+                          new_with_vtable=0)  # all virtual
 
     def test_cond_call_multiple_in_optimizer_1(self):
         # test called several times with the same arguments, but

From pypy.commits at gmail.com  Thu Nov 24 10:47:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 07:47:52 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Use conditional_call_value
Message-ID: <58370ba8.12921c0a.57a99.0c2d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88640:85f1c9ac66f6
Date: 2016-11-24 15:09 +0100
http://bitbucket.org/pypy/pypy/changeset/85f1c9ac66f6/

Log:	Use conditional_call_value

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -79,14 +79,9 @@
 
     def identifier_w(self, space):
         try:
-            if jit.isconstant(self._value):
-                # constantly encode that unicode string; don't try
-                # to access the cache _utf8 at all
-                identifier = self._value.encode('utf-8')
-            else:
-                # call the elidable function, with a jit.call_shortcut in case
-                # self._utf8 is already computed
-                identifier = g_identifier_w(self, space)
+            identifier = jit.conditional_call_elidable(
+                                self._utf8, g_encode_utf8, self._value)
+            self._utf8 = identifier
         except UnicodeEncodeError:
             # bah, this is just to get an official app-level
             # UnicodeEncodeError
@@ -1278,15 +1273,9 @@
                                      allow_surrogates=allow_surrogates)
 
 @jit.elidable
- at jit.call_shortcut
-def g_identifier_w(self, space):
-    """This is a global function because of @jit.call_shortcut"""
-    identifier = self._utf8
-    if identifier is not None:
-        return identifier
-    identifier = self._value.encode('utf-8')
-    self._utf8 = identifier
-    return identifier
+def g_encode_utf8(value):
+    """This is a global function because of jit.conditional_call_value"""
+    return value.encode('utf-8')
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')

From pypy.commits at gmail.com  Thu Nov 24 11:12:14 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 08:12:14 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5837115e.46831c0a.c8125.1900@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88641:84e8172fc396
Date: 2016-11-24 17:11 +0100
http://bitbucket.org/pypy/pypy/changeset/84e8172fc396/

Log:	hg merge default

diff too long, truncating to 2000 out of 2923 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -67,6 +67,7 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
     g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -113,7 +113,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -127,7 +129,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -396,7 +400,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -417,7 +421,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -127,6 +127,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -886,3 +892,19 @@
         assert "keywords must be strings" in str(e.value)
         e = raises(TypeError, "f(y=2, **{'x': 5}, x=6)")
         assert "got multiple values for keyword argument 'x'" in str(e.value)
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -62,12 +62,14 @@
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
+    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.ensure_object_strategy()  # make sure we can return a borrowed obj
-    # XXX ^^^ how does this interact with CPyListStrategy?
+    cpy_strategy = space.fromcache(CPyListStrategy)
+    if w_list.strategy is not cpy_strategy:
+        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
     w_res = w_list.getitem(index)
     return w_res     # borrowed ref
 
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -111,11 +111,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.bytesobject import PyString_AsString
+    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -260,7 +260,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.wrap('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_ASCII(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -203,3 +214,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -285,6 +285,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -306,12 +306,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -467,7 +467,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -725,14 +725,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_sequence:
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -11,58 +11,48 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def unicode_iter(space):
     "Utility that returns the app-level descriptor str.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_unicode, '__iter__')
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    return space.lookup_in_type(space.w_dict, '__getitem__')
+
 
 def raiseattrerror(space, w_obj, w_name, w_descr=None):
     # space.repr always returns an encodable string.
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -339,12 +339,18 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -374,6 +374,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -671,6 +675,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 
@@ -700,4 +708,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -22,7 +22,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -50,6 +51,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -202,6 +206,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -348,6 +358,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -392,6 +404,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -482,6 +498,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -326,7 +327,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -789,6 +789,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -811,6 +815,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -663,6 +664,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return compute_hash(hlstr(s))
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return compute_hash(hlunicode(u))
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,9 +9,9 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
@@ -347,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -595,33 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
-        descr = op.getdescr()
-        if descr is None:
-            return
-        assert isinstance(descr, CallDescr)
-        effectinfo = descr.get_extra_info()
-        if effectinfo is None or effectinfo.call_shortcut is None:
-            return
-        if op.type == 'r':
-            cond_call_opnum = rop.COND_CALL_VALUE_R
-        elif op.type == 'i':
-            cond_call_opnum = rop.COND_CALL_VALUE_I
-        else:
-            return
-        cs = effectinfo.call_shortcut
-        ptr_box = op.getarg(1 + cs.argnum)
-        if cs.fielddescr is not None:
-            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                           raw=(ptr_box.type == 'i'))
-        else:
-            value_box = ptr_box
-        self.replace_op_with(op, ResOperation(cond_call_opnum,
-                                              [value_box] + op.getarglist(),
-                                              descr=descr))
-
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,8 +1,7 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr, get_call_descr
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -81,21 +80,6 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
-        call_shortcut = CallShortcut(0, tzdescr)
-        effectinfo = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut)
-        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
-            [lltype.Ptr(T)], lltype.Signed,
-            effectinfo)
-        call_shortcut_2 = CallShortcut(0, None)
-        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut_2)
-        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
-            [lltype.Signed], lltype.Signed,
-            effectinfo_2)
-        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -216,7 +200,6 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
-    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1239,6 +1222,10 @@
                        'i3 = gc_load_i(p0,'
                                  '%(strlendescr.offset)s,'
                                  '%(strlendescr.field_size)s)'],
+        [True,  (1,),  'i3 = strhash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(strhashdescr.offset)s,'
+                                 '-%(strhashdescr.field_size)s)'],
         #[False, (1,),  'i3 = unicodelen(p0)' '->'
         #               'i3 = gc_load_i(p0,'
         #                       '%(unicodelendescr.offset)s,'
@@ -1247,7 +1234,10 @@
                        'i3 = gc_load_i(p0,'
                                '%(unicodelendescr.offset)s,'
                                '%(unicodelendescr.field_size)s)'],
-
+        [True,  (1,),  'i3 = unicodehash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(unicodehashdescr.offset)s,'
+                                 '-%(unicodehashdescr.field_size)s)'],
         ## getitem str/unicode
         [True,  (2,4), 'i3 = unicodegetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,'
@@ -1446,26 +1436,3 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
-
-    def test_handle_call_shortcut(self):
-        self.check_rewrite("""
-            [p0]
-            i1 = call_i(123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """, """
-            [p0]
-            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
-            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """)
-
-    def test_handle_call_shortcut_2(self):
-        self.check_rewrite("""
-            [i0]
-            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """, """
-            [i0]
-            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,7 +16,6 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
-    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -656,10 +656,12 @@
     _COND_CALL_SAVE_REGS = [r.r3, r.r4, r.r5, r.r6, r.r12]
 
     def emit_cond_call(self, op, arglocs, regalloc):
+        resloc = arglocs[0]
+        arglocs = arglocs[1:]
+
         fcond = self.guard_success_cc
         self.guard_success_cc = c.cond_none
         assert fcond != c.cond_none
-        fcond = c.negate(fcond)
 
         jmp_adr = self.mc.get_relative_pos()
         self.mc.trap()        # patched later to a 'bc'
@@ -691,7 +693,10 @@
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.bl_abs(cond_call_adr)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  If we have a result, move
+        # it from r2 to its expected location.
+        if resloc is not None:
+            self.mc.mr(resloc.value, r.SCRATCH2.value)
         relative_target = self.mc.currpos() - jmp_adr
         pmc = OverwritingBuilder(self.mc, jmp_adr, 1)
         BI, BO = c.encoding[fcond]
@@ -701,6 +706,9 @@
         # guard_no_exception too
         self.previous_cond_call_jcond = jmp_adr, BI, BO
 
+    emit_cond_call_value_i = emit_cond_call
+    emit_cond_call_value_r = emit_cond_call
+
 
 class FieldOpAssembler(object):
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -315,6 +315,7 @@
         #   * r2 is the gcmap
         #   * the old value of these regs must already be stored in the jitframe
         #   * on exit, all registers are restored from the jitframe
+        #   * the result of the call, if any, is moved to r2
 
         mc = PPCBuilder()
         self.mc = mc
@@ -347,7 +348,11 @@
         # Finish
         self._reload_frame_if_necessary(mc)
 
+        # Move the result, if any, to r2
+        mc.mr(r.SCRATCH2.value, r.r3.value)
+
         mc.mtlr(r.RCS1.value)     # restore LR
+
         self._pop_core_regs_from_jitframe(mc, saved_regs)
         if supports_floats:
             self._pop_fp_regs_from_jitframe(mc)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1038,14 +1038,34 @@
 
     def prepare_cond_call(self, op):
         self.load_condition_into_cc(op.getarg(0))
-        locs = []
+        self.assembler.guard_success_cc = c.negate(
+            self.assembler.guard_success_cc)
+        # ^^^ if arg0==0, we jump over the next block of code (the call)
+        locs = [None]
         # support between 0 and 4 integer arguments
         assert 2 <= op.numargs() <= 2 + 4
         for i in range(1, op.numargs()):
             loc = self.loc(op.getarg(i))
             assert loc.type != FLOAT
             locs.append(loc)
-        return locs
+        return locs     # [None, function, args...]
+
+    def prepare_cond_call_value_i(self, op):
+        x = self.ensure_reg(op.getarg(0))
+        self.load_condition_into_cc(op.getarg(0))
+        self.rm.force_allocate_reg(op, selected_reg=x)   # spilled if survives
+        # ^^^ if arg0!=0, we jump over the next block of code (the call)
+        locs = [x]
+        # support between 0 and 4 integer arguments
+        assert 2 <= op.numargs() <= 2 + 4
+        for i in range(1, op.numargs()):
+            loc = self.loc(op.getarg(i))
+            assert loc.type != FLOAT
+            locs.append(loc)
+        return locs     # [res, function, args...]
+
+    prepare_cond_call_value_r = prepare_cond_call_value_i
+
 
 def notimplemented(self, op):
     msg = '[PPC/regalloc] %s not implemented\n' % op.getopname()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,9 +2448,6 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
-        if not self.cpu.supports_cond_call_value:
-            py.test.skip("missing supports_cond_call_value")
-
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -711,11 +711,6 @@
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
 
-    def filter(self, builder):
-        if not builder.cpu.supports_cond_call_value and \
-           self.opnum == rop.COND_CALL_VALUE_I:
-            raise test_random.CannotProduceOperation
-
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -20,11 +20,6 @@
     r = Random()
     r.jumpahead(piece*99999999)
     OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,7 +16,6 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
-    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.CallTest):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,10 +7,9 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection, CallShortcut)
+    EffectInfo, CallInfoCollection)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
-from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -215,7 +214,6 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
-        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -230,12 +228,6 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
-            if hasattr(funcobj, 'graph'):
-                call_shortcut = self.find_call_shortcut(funcobj.graph)
-            if getattr(func, "_call_shortcut_", False):
-                assert call_shortcut is not None, (
-                    "%r: marked as @jit.call_shortcut but shortcut not found"
-                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -250,8 +242,6 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
-                if hasattr(graph.func, '_call_shortcut_'):
-                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -308,7 +298,6 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
-            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -379,76 +368,3 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
-
-    def find_call_shortcut(self, graph):
-        """Identifies graphs that start like this:
-
-           def graph(x, y, z):         def graph(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r: return r
-        """
-        block = graph.startblock
-        operations = block.operations
-        c_fieldname = None
-        if not operations:
-            v_inst = v_result = block.exitswitch
-        else:
-            op = operations[0]
-            if len(op.args) == 0:
-                return
-            if op.opname != 'getfield':  # check for this form:
-                v_inst = op.args[0]      #     if y is not None;
-                v_result = v_inst        #          return y
-            else:
-                operations = operations[1:]
-                [v_inst, c_fieldname] = op.args
-                v_result = op.result
-        if not isinstance(v_inst, Variable):
-            return
-        if v_result.concretetype != graph.getreturnvar().concretetype:
-            return
-        if v_result.concretetype == lltype.Void:
-            return
-        argnum = i = 0
-        while block.inputargs[i] is not v_inst:
-            if block.inputargs[i].concretetype != lltype.Void:
-                argnum += 1
-            i += 1
-        PSTRUCT = v_inst.concretetype
-        v_check = v_result
-        fastcase = True
-        for op in operations:
-            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
-                    and v_check is op.args[0]):
-                v_check = op.result
-            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
-                v_check = op.result
-                fastcase = not fastcase
-            elif (op.opname in ('int_eq', 'int_ne')
-                    and v_check is op.args[0]
-                    and isinstance(op.args[1], Constant)
-                    and op.args[1].value == 0):
-                v_check = op.result
-                if op.opname == 'int_eq':
-                    fastcase = not fastcase
-            else:
-                return
-        if v_check.concretetype is not lltype.Bool:
-            return
-        if block.exitswitch is not v_check:
-            return
-
-        links = [link for link in block.exits if link.exitcase == fastcase]
-        if len(links) != 1:
-            return
-        [link] = links
-        if link.args != [v_result]:
-            return
-        if not link.target.is_final_block():
-            return
-
-        if c_fieldname is not None:
-            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
-        else:
-            fielddescr = None
-        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,8 +117,7 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True,
-                call_shortcut=None):
+                can_collect=True):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -136,8 +135,7 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect,
-               call_shortcut)
+               can_collect)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -192,7 +190,6 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
-        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -278,8 +275,7 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True,
-                                 call_shortcut=None):
+                                 can_collect=True):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -368,8 +364,7 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect,
-                      call_shortcut)
+                      can_collect)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -392,24 +387,6 @@
 
 # ____________________________________________________________
 
-
-class CallShortcut(object):
-    def __init__(self, argnum, fielddescr):
-        self.argnum = argnum
-        self.fielddescr = fielddescr
-
-    def __eq__(self, other):
-        return (isinstance(other, CallShortcut) and
-                self.argnum == other.argnum and
-                self.fielddescr == other.fielddescr)
-    def __ne__(self, other):
-        return not (self == other)
-    def __hash__(self):
-        return hash((self.argnum, self.fielddescr))
-
-# ____________________________________________________________
-
-
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -789,12 +789,20 @@
                                                 arrayfielddescr,
                                                 arraydescr)
             return []
+        # check for the string or unicode hash field
+        STRUCT = v_inst.concretetype.TO
+        if STRUCT == rstr.STR:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('strhash', [v_inst], op.result)
+        elif STRUCT == rstr.UNICODE:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('unicodehash', [v_inst], op.result)
         # check for _immutable_fields_ hints
-        immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+        immut = STRUCT._immutable_field(c_fieldname.value)
         need_live = False
         if immut:
             if (self.callcontrol is not None and
-                self.callcontrol.could_be_green_field(v_inst.concretetype.TO,
+                self.callcontrol.could_be_green_field(STRUCT,
                                                       c_fieldname.value)):
                 pure = '_greenfield'
                 need_live = True
@@ -802,10 +810,9 @@
                 pure = '_pure'
         else:
             pure = ''
-        self.check_field_access(v_inst.concretetype.TO)
-        argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
-        descr = self.cpu.fielddescrof(v_inst.concretetype.TO,
-                                      c_fieldname.value)
+        self.check_field_access(STRUCT)
+        argname = getattr(STRUCT, '_gckind', 'gc')
+        descr = self.cpu.fielddescrof(STRUCT, c_fieldname.value)
         kind = getkind(RESULT)[0]
         if argname != 'gc':
             assert argname == 'raw'
@@ -822,7 +829,7 @@
         if immut in (IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY):
             op1.opname += "_pure"
             descr1 = self.cpu.fielddescrof(
-                v_inst.concretetype.TO,
+                STRUCT,
                 quasiimmut.get_mutate_field_name(c_fieldname.value))
             return [SpaceOperation('-live-', [], None),
                    SpaceOperation('record_quasiimmut_field',
@@ -1569,7 +1576,7 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
+    def _rewrite_op_cond_call(self, op, rewritten_opname):
         have_floats = False
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
@@ -1580,13 +1587,18 @@
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
-        op1 = self.rewrite_call(op, 'conditional_call',
+        op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
                                 calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_conditional_call(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call')
+    def rewrite_op_jit_conditional_call_value(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call_value')
+
     def handle_jit_marker__jit_merge_point(self, op, jitdriver):
         assert self.portal_jd is not None, (
             "'jit_merge_point' in non-portal graph!")
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+from rpython.jit.codewriter.effectinfo import EffectInfo
 
 
 class FakePolicy:
@@ -368,121 +368,3 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
-
-def test_find_call_shortcut():
-    class FakeCPU:
-        def fielddescrof(self, TYPE, fieldname):
-            if isinstance(TYPE, lltype.GcStruct):
-                if fieldname == 'inst_foobar':
-                    return 'foobardescr'
-                if fieldname == 'inst_fooref':
-                    return 'foorefdescr'
-            if TYPE == RAW and fieldname == 'x':
-                return 'xdescr'
-            assert False, (TYPE, fieldname)
-    cc = CallControl(FakeCPU())
-
-    class B(object):
-        foobar = 0
-        fooref = None
-
-    def f1(a, b, c):
-        if b.foobar:
-            return b.foobar
-        b.foobar = a + c
-        return b.foobar
-
-    def f2(x, y, z, b):
-        r = b.fooref
-        if r is not None:
-            return r
-        r = b.fooref = B()
-        return r
-
-    class Space(object):
-        def _freeze_(self):
-            return True
-    space = Space()
-
-    def f3(space, b):
-        r = b.foobar
-        if not r:
-            r = b.foobar = 123
-        return r
-
-    def f4(raw):
-        r = raw.x
-        if r != 0:
-            return r
-        raw.x = 123
-        return 123
-    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
-
-    def f5(b):
-        r = b.foobar
-        if r == 0:
-            r = b.foobar = 123
-        return r
-
-    def f6(b):
-        if b is not None:
-            return b
-        return B()
-
-    def f7(c, a):
-        if a:
-            return a
-        return 123
-
-    def b_or_none(c):
-        if c > 15:
-            return B()
-        return None
-
-    def f(a, c):
-        b = B()
-        f1(a, b, c)
-        f2(a, c, a, b)
-        f3(space, b)
-        r = lltype.malloc(RAW, flavor='raw')
-        f4(r)
-        f5(b)
-        f6(b_or_none(c))
-        f7(c, a)
-
-    rtyper = support.annotate(f, [10, 20])
-    f1_graph = rtyper.annotator.translator._graphof(f1)
-    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
-    f2_graph = rtyper.annotator.translator._graphof(f2)
-    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
-    f3_graph = rtyper.annotator.translator._graphof(f3)
-    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
-    f4_graph = rtyper.annotator.translator._graphof(f4)
-    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
-    f5_graph = rtyper.annotator.translator._graphof(f5)
-    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
-    f6_graph = rtyper.annotator.translator._graphof(f6)
-    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
-    f7_graph = rtyper.annotator.translator._graphof(f7)
-    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
-
-def test_cant_find_call_shortcut():
-    from rpython.jit.backend.llgraph.runner import LLGraphCPU
-
-    @jit.dont_look_inside
-    @jit.call_shortcut
-    def f1(n):
-        return n + 17   # no call shortcut found
-
-    def f(n):
-        return f1(n)
-
-    rtyper = support.annotate(f, [1])
-    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
-    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
-    res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
-    call_op = f_graph.startblock.operations[0]
-    assert call_op.opname == 'direct_call'
-    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
-    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1199,13 +1199,27 @@
     def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr):
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
-    # conditional calls - note that they cannot return stuff
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
+        # conditional calls - condition is a flag, and they cannot return stuff
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
+    @arguments("cpu", "i", "i", "I", "R", "d", returns="i")
+    def bhimpl_conditional_call_value_ir_i(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if value == 0:
+            value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
+        return value
+
+    @arguments("cpu", "r", "i", "I", "R", "d", returns="r")
+    def bhimpl_conditional_call_value_ir_r(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if not value:
+            value = cpu.bh_call_r(func, args_i, args_r, None, calldescr)
+        return value
+
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
@@ -1493,6 +1507,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_strhash(cpu, string):
+        return cpu.bh_strhash(string)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1509,6 +1526,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_unicodehash(cpu, unicode):
+        return cpu.bh_unicodehash(unicode)
 
     @arguments("i", "i")
     def bhimpl_rvmprof_code(leaving, unique_id):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,6 +271,7 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
+            OpHelpers.is_cond_call_value(opnum) or
             opnum == rop.COND_CALL):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,5 +1,5 @@
 
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
@@ -73,6 +73,9 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
+    def getstrhash(self, op, mode):
+        return None
+
     def copy_fields_to_const(self, constinfo, optheap):
         pass
 
@@ -790,6 +793,20 @@
                 return None
             return ConstInt(len(s))
 
+    def getstrhash(self, op, mode):
+        from rpython.jit.metainterp.optimizeopt import vstring
+
+        if mode is vstring.mode_string:
+            s = self._unpack_str(vstring.mode_string)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+        else:
+            s = self._unpack_str(vstring.mode_unicode)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -399,9 +399,9 @@
 
     def optimize_INT_EQ(self, op):
         arg0 = self.get_box_replacement(op.getarg(0))
+        b1 = self.getintbound(arg0)
         arg1 = self.get_box_replacement(op.getarg(1))
-        b1 = self.getintbound(op.getarg(0))
-        b2 = self.getintbound(op.getarg(1))
+        b2 = self.getintbound(arg1)
         if b1.known_gt(b2):
             self.make_constant_int(op, 0)
         elif b1.known_lt(b2):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -261,9 +261,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
-    def _can_optimize_call_pure(self, op):
+    def _can_optimize_call_pure(self, op, start_index=0):
         arg_consts = []
-        for i in range(op.numargs()):
+        for i in range(start_index, op.numargs()):
             arg = op.getarg(i)
             const = self.optimizer.get_constant_box(arg)
             if const is None:
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -151,13 +151,13 @@
             self._pure_operations[opnum] = recentops = RecentPureOps()
         return recentops
 
-    def optimize_CALL_PURE_I(self, op):
+    def optimize_call_pure(self, op, start_index=0):
         # Step 1: check if all arguments are constant
-        for arg in op.getarglist():
-            self.optimizer.force_box(arg)
+        for i in range(start_index, op.numargs()):
+            self.optimizer.force_box(op.getarg(i))
             # XXX hack to ensure that virtuals that are
             #     constant are presented that way
-        result = self._can_optimize_call_pure(op)
+        result = self._can_optimize_call_pure(op, start_index=start_index)
         if result is not None:
             # this removes a CALL_PURE with all constant arguments.
             self.make_constant(op, result)
@@ -168,32 +168,46 @@
         # CALL_PURE.
         for pos in self.call_pure_positions:
             old_op = self.optimizer._newoperations[pos]
-            if self.optimize_call_pure(op, old_op):
+            if self.optimize_call_pure_old(op, old_op, start_index):
                 return
         if self.extra_call_pure:
             for i, old_op in enumerate(self.extra_call_pure):
-                if self.optimize_call_pure(op, old_op):
+                if self.optimize_call_pure_old(op, old_op, start_index):
                     if isinstance(old_op, PreambleOp):
                         old_op = self.optimizer.force_op_from_preamble(old_op)
                         self.extra_call_pure[i] = old_op
                     return
 
-        # replace CALL_PURE with just CALL
-        opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = self.optimizer.replace_op_with(op, opnum)
+        # replace CALL_PURE with just CALL (but keep COND_CALL_VALUE)
+        if start_index == 0:
+            opnum = OpHelpers.call_for_descr(op.getdescr())
+            newop = self.optimizer.replace_op_with(op, opnum)
+        else:
+            newop = op
         return self.emit_result(CallPureOptimizationResult(self, newop))
 
+    def optimize_CALL_PURE_I(self, op):
+        return self.optimize_call_pure(op)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
-    def optimize_call_pure(self, op, old_op):
-        if (op.numargs() != old_op.numargs() or
-            op.getdescr() is not old_op.getdescr()):
+    def optimize_COND_CALL_VALUE_I(self, op):
+        return self.optimize_call_pure(op, start_index=1)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
+    def optimize_call_pure_old(self, op, old_op, start_index):
+        if op.getdescr() is not old_op.getdescr():
             return False
-        for i, box in enumerate(old_op.getarglist()):
-            if not self.get_box_replacement(op.getarg(i)).same_box(box):
+        # this will match a call_pure and a cond_call_value with
+        # the same function and arguments
+        j = start_index
+        old_start_index = OpHelpers.is_cond_call_value(old_op.opnum)
+        for i in range(old_start_index, old_op.numargs()):
+            box = old_op.getarg(i)
+            if not self.get_box_replacement(op.getarg(j)).same_box(box):
                 break
+            j += 1
         else:
             # all identical
             # this removes a CALL_PURE that has the same (non-constant)
@@ -250,10 +264,17 @@
             # don't move call_pure_with_exception in the short preamble...
             # issue #2015
 
+            # Also, don't move cond_call_value in the short preamble.
+            # The issue there is that it's usually pointless to try to
+            # because the 'value' argument is typically not a loop
+            # invariant, and would really need to be in order to end up
+            # in the short preamble.  Maybe the code works anyway in the
+            # other rare case, but better safe than sorry and don't try.
             effectinfo = op.getdescr().get_extra_info()
             if not effectinfo.check_can_raise(ignore_memoryerror=True):
                 assert rop.is_call(op.opnum)
-                sb.add_pure_op(op)
+                if not OpHelpers.is_cond_call_value(op.opnum):
+                    sb.add_pure_op(op)
 
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -596,6 +596,19 @@
             op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         return self.emit(op)
 
+    def optimize_COND_CALL_VALUE_I(self, op):
+        # look if we know the nullness of the first argument
+        info = self.getnullness(op.getarg(0))
+        if info == INFO_NONNULL:
+            self.make_equal_to(op, op.getarg(0))
+            self.last_emitted_operation = REMOVED
+            return
+        if info == INFO_NULL:
+            opnum = OpHelpers.call_pure_for_type(op.type)
+            op = self.replace_op_with(op, opnum, args=op.getarglist()[1:])
+        return self.emit(op)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
     def _optimize_nullness(self, op, box, expect_nonnull):
         info = self.getnullness(box)
         if info == INFO_NONNULL:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, TreeLoop
+from rpython.jit.metainterp.history import ConstPtr
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import build_opt_chain
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
@@ -8674,6 +8675,75 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_with_a_constant_i(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(0, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i2 = call_i(123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_with_a_constant_i2(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(12, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        escape_n(12)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r1(self):
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        jump(p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_cond_call_r2(self):
+        ops = """
+        [p1]
+        guard_nonnull(p1) []
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        guard_nonnull(p1) []
+        p3 = escape_r(p1)
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r3(self):
+        arg_consts = [ConstInt(i) for i in (123, 4, 5, 6)]
+        call_pure_results = {tuple(arg_consts): ConstPtr(self.myptr)}
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, 4, 5, 6, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        p3 = escape_r(ConstPtr(myptr))
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -438,6 +438,10 @@
                     oopspecindex=EffectInfo.OS_INT_PY_MOD)
     int_py_mod_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
 
+    FUNC = lltype.FuncType([], llmemory.GCREF)
+    ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE)
+    plain_r_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
+
     namespace = locals()
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -537,6 +537,20 @@
                 return
         return self.emit(op)
 
+    def optimize_STRHASH(self, op):
+        return self._optimize_STRHASH(op, mode_string)
+    def optimize_UNICODEHASH(self, op):
+        return self._optimize_STRHASH(op, mode_unicode)
+
+    def _optimize_STRHASH(self, op, mode):
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            lgtop = opinfo.getstrhash(op, mode)
+            if lgtop is not None:
+                self.make_equal_to(op, lgtop)
+                return
+        return self.emit(op)
+
     def optimize_COPYSTRCONTENT(self, op):
         return self._optimize_COPYSTRCONTENT(op, mode_string)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1059,8 +1059,21 @@
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
+        if isinstance(condbox, ConstInt) and condbox.value == 0:
+            return   # so that the heapcache can keep argboxes virtual
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
+    @arguments("box", "box", "boxes2", "descr", "orgpc")
+    def _opimpl_conditional_call_value(self, valuebox, funcbox, argboxes,
+                                       calldescr, pc):
+        if isinstance(valuebox, Const) and valuebox.nonnull():
+            return valuebox
+        return self.do_conditional_call(valuebox, funcbox, argboxes,
+                                        calldescr, pc, is_value=True)
+
+    opimpl_conditional_call_value_ir_i = _opimpl_conditional_call_value
+    opimpl_conditional_call_value_ir_r = _opimpl_conditional_call_value
+
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
@@ -1150,6 +1163,20 @@
         self.execute(rop.UNICODESETITEM, unicodebox, indexbox, newcharbox)
 
     @arguments("box")
+    def opimpl_strhash(self, strbox):
+        if isinstance(strbox, ConstPtr):
+            h = self.metainterp.cpu.bh_strhash(strbox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.STRHASH, strbox)
+
+    @arguments("box")
+    def opimpl_unicodehash(self, unicodebox):
+        if isinstance(unicodebox, ConstPtr):
+            h = self.metainterp.cpu.bh_unicodehash(unicodebox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.UNICODEHASH, unicodebox)
+
+    @arguments("box")
     def opimpl_newstr(self, lengthbox):
         return self.execute(rop.NEWSTR, lengthbox)
 
@@ -1538,7 +1565,7 @@
                                                             descr=descr)
         if pure and not self.metainterp.last_exc_value and op:
             op = self.metainterp.record_result_of_call_pure(op, argboxes, descr,
-                patch_pos)
+                patch_pos, opnum)
             exc = exc and not isinstance(op, Const)
         if exc:
             if op is not None:
@@ -1712,16 +1739,31 @@
             else:
                 assert False
 
-    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
-        if isinstance(condbox, ConstInt) and condbox.value == 0:
-            return   # so that the heapcache can keep argboxes virtual
+    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc,
+                            is_value=False):
         allboxes = self._build_allboxes(funcbox, argboxes, descr)
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop.COND_CALL, [condbox] + allboxes, descr,
-                                    exc, pure)
+        allboxes = [condbox] + allboxes
+        # COND_CALL cannot be pure (=elidable): it has no result.
+        # On the other hand, COND_CALL_VALUE is always calling a pure

From pypy.commits at gmail.com  Thu Nov 24 11:53:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 08:53:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Potential code size reduction: move a
 bit of the logic out, which
Message-ID: <58371af4.542e1c0a.d6cc.2872@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88642:50e4baa0921b
Date: 2016-11-24 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/50e4baa0921b/

Log:	Potential code size reduction: move a bit of the logic out, which is
	repeated many times over

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -132,20 +132,8 @@
                 elif argtype == 'I' or argtype == 'R' or argtype == 'F':
                     assert argcodes[next_argcode] == argtype
                     next_argcode = next_argcode + 1
-                    length = ord(code[position])
-                    position += 1
-                    value = []
-                    for i in range(length):
-                        index = ord(code[position+i])
-                        if   argtype == 'I': reg = self.registers_i[index]
-                        elif argtype == 'R': reg = self.registers_r[index]
-                        elif argtype == 'F': reg = self.registers_f[index]
-                        if not we_are_translated():
-                            assert not isinstance(reg, MissingValue), (
-                                name, self.jitcode, position)
-                        value.append(reg)
-                    make_sure_not_resized(value)
-                    position += length
+                    value = self._get_list_of_values(code, position, argtype)
+                    position += 1 + len(value)
                 elif argtype == 'self':
                     value = self
                 elif argtype == 'cpu':
@@ -1657,6 +1645,24 @@
             if box is not None:
                 self.setarg_f(i, box.getfloatstorage())
 
+    @specialize.arg(3)
+    def _get_list_of_values(self, code, position, argtype):
+        length = ord(code[position])
+        position += 1
+        value = []
+        for i in range(length):
+            index = ord(code[position+i])
+            if   argtype == 'I': reg = self.registers_i[index]
+            elif argtype == 'R': reg = self.registers_r[index]
+            elif argtype == 'F': reg = self.registers_f[index]
+            else: assert 0
+            if not we_are_translated():
+                assert not isinstance(reg, MissingValue), (
+                    name, self.jitcode, position)
+            value.append(reg)
+        make_sure_not_resized(value)
+        return value
+
 # ____________________________________________________________
 
 def _run_forever(blackholeinterp, current_exc):

From pypy.commits at gmail.com  Thu Nov 24 11:59:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 08:59:08 -0800 (PST)
Subject: [pypy-commit] pypy default: Test and fix: try harder to propagate
 'can_be_None=False' information
Message-ID: <58371c5c.c4811c0a.2dd69.2a32@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88643:27b352d10c03
Date: 2016-11-24 17:58 +0100
http://bitbucket.org/pypy/pypy/changeset/27b352d10c03/

Log:	Test and fix: try harder to propagate 'can_be_None=False'
	information

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1243,7 +1243,13 @@
                                                  args_s[1], args_s[2:])
         if self.instance == _jit_conditional_call_value:
             from rpython.annotator import model as annmodel
-            return annmodel.unionof(s_res, args_s[0])
+            # the result is either s_res, i.e. the function result, or
+            # it is args_s[0]-but-not-none.  The "not-none" part is
+            # only useful for pointer-like types, but it means that
+            # args_s[0] could be NULL without the result of the whole
+            # conditional_call_elidable() necessarily returning a result
+            # that can be NULL.
+            return annmodel.unionof(s_res, args_s[0].nonnoneify())
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -314,6 +314,24 @@
         res = self.interpret(f, [-42, 1000, 100])
         assert res == -42
 
+    def test_conditional_call_elidable_annotates_nonnull(self):
+        class X:
+            pass
+        def g(n):
+            return X()   # non-null
+        def f(x, n):
+            y = conditional_call_elidable(x, g, n)
+            return y     # now, y is known to be non-null, even if x can be
+        def main(n):
+            if n > 100:
+                x = X()
+            else:
+                x = None
+            return f(x, n)
+        t = TranslationContext()
+        s = t.buildannotator().build_types(main, [int])
+        assert s.can_be_None is False
+
     def test_enter_leave_portal_frame(self):
         from rpython.translator.interactive import Translation
         def g():

From pypy.commits at gmail.com  Thu Nov 24 11:59:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 08:59:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58371c6c.c19d1c0a.22254.2d36@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88644:a016f52580ec
Date: 2016-11-24 17:58 +0100
http://bitbucket.org/pypy/pypy/changeset/a016f52580ec/

Log:	hg merge default

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -132,20 +132,8 @@
                 elif argtype == 'I' or argtype == 'R' or argtype == 'F':
                     assert argcodes[next_argcode] == argtype
                     next_argcode = next_argcode + 1
-                    length = ord(code[position])
-                    position += 1
-                    value = []
-                    for i in range(length):
-                        index = ord(code[position+i])
-                        if   argtype == 'I': reg = self.registers_i[index]
-                        elif argtype == 'R': reg = self.registers_r[index]
-                        elif argtype == 'F': reg = self.registers_f[index]
-                        if not we_are_translated():
-                            assert not isinstance(reg, MissingValue), (
-                                name, self.jitcode, position)
-                        value.append(reg)
-                    make_sure_not_resized(value)
-                    position += length
+                    value = self._get_list_of_values(code, position, argtype)
+                    position += 1 + len(value)
                 elif argtype == 'self':
                     value = self
                 elif argtype == 'cpu':
@@ -1657,6 +1645,24 @@
             if box is not None:
                 self.setarg_f(i, box.getfloatstorage())
 
+    @specialize.arg(3)
+    def _get_list_of_values(self, code, position, argtype):
+        length = ord(code[position])
+        position += 1
+        value = []
+        for i in range(length):
+            index = ord(code[position+i])
+            if   argtype == 'I': reg = self.registers_i[index]
+            elif argtype == 'R': reg = self.registers_r[index]
+            elif argtype == 'F': reg = self.registers_f[index]
+            else: assert 0
+            if not we_are_translated():
+                assert not isinstance(reg, MissingValue), (
+                    name, self.jitcode, position)
+            value.append(reg)
+        make_sure_not_resized(value)
+        return value
+
 # ____________________________________________________________
 
 def _run_forever(blackholeinterp, current_exc):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1243,7 +1243,13 @@
                                                  args_s[1], args_s[2:])
         if self.instance == _jit_conditional_call_value:
             from rpython.annotator import model as annmodel
-            return annmodel.unionof(s_res, args_s[0])
+            # the result is either s_res, i.e. the function result, or
+            # it is args_s[0]-but-not-none.  The "not-none" part is
+            # only useful for pointer-like types, but it means that
+            # args_s[0] could be NULL without the result of the whole
+            # conditional_call_elidable() necessarily returning a result
+            # that can be NULL.
+            return annmodel.unionof(s_res, args_s[0].nonnoneify())
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -314,6 +314,24 @@
         res = self.interpret(f, [-42, 1000, 100])
         assert res == -42
 
+    def test_conditional_call_elidable_annotates_nonnull(self):
+        class X:
+            pass
+        def g(n):
+            return X()   # non-null
+        def f(x, n):
+            y = conditional_call_elidable(x, g, n)
+            return y     # now, y is known to be non-null, even if x can be
+        def main(n):
+            if n > 100:
+                x = X()
+            else:
+                x = None
+            return f(x, n)
+        t = TranslationContext()
+        s = t.buildannotator().build_types(main, [int])
+        assert s.can_be_None is False
+
     def test_enter_leave_portal_frame(self):
         from rpython.translator.interactive import Translation
         def g():

From pypy.commits at gmail.com  Thu Nov 24 13:19:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 24 Nov 2016 10:19:57 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: hg merge default
Message-ID: <58372f4d.45f6c20a.f25f.87ca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88645:9b3564c820ef
Date: 2016-11-24 06:07 +0000
http://bitbucket.org/pypy/pypy/changeset/9b3564c820ef/

Log:	hg merge default

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -111,7 +111,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -125,7 +127,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -355,7 +359,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -374,7 +378,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,6 +120,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type_where(self, cls, name):
+        return 'hopefully not needed', getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -805,3 +811,19 @@
             assert str(e) == "myerror"
         else:
             assert False, "Expected TypeError"
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -495,11 +495,10 @@
             if space.isinstance_w(w_data, space.w_unicode):
                 # note: "encode" is called before we acquire the lock
                 # for this file, which is done in file_write_str().
-                # Direct access to unicodeobject because we don't want
+                # Use a specific space method because we don't want
                 # to call user-defined "encode" methods here.
-                from pypy.objspace.std.unicodeobject import encode_object
-                w_data = encode_object(space, w_data, self.encoding,
-                                       self.errors)
+                w_data = space.encode_unicode_object(w_data,
+                     self.encoding, self.errors)
             data = space.charbuf_w(w_data)
         self.file_write_str(data)
 
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -293,7 +293,7 @@
 same_attributes_as_in_file.remove('__init__')
 same_attributes_as_in_file.extend([
     'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace',
-    'writelines', '__exit__', '__weakref__'])
+    'writelines', '__exit__', '__weakref__', 'write'])
 
 W_BZ2File.typedef = TypeDef(
     "BZ2File",
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -252,7 +252,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.wrap('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_Unicode(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -199,3 +210,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -312,6 +312,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -331,12 +331,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -469,7 +469,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -852,14 +852,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -61,16 +61,24 @@
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_str,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
+                                                  '__getitem__')
+    return w_getitem
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__getitem__')
-    return w_iter
+    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
+                                                  '__getitem__')
+    return w_getitem
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
+                                                  '__getitem__')
+    return w_getitem
+
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -330,6 +330,9 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -626,6 +626,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -22,7 +22,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -51,6 +52,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -203,6 +207,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -358,6 +368,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -402,6 +414,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -492,6 +508,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -399,9 +399,9 @@
 
     def optimize_INT_EQ(self, op):
         arg0 = self.get_box_replacement(op.getarg(0))
+        b1 = self.getintbound(arg0)
         arg1 = self.get_box_replacement(op.getarg(1))
-        b1 = self.getintbound(op.getarg(0))
-        b2 = self.getintbound(op.getarg(1))
+        b2 = self.getintbound(arg1)
         if b1.known_gt(b2):
             self.make_constant_int(op, 0)
         elif b1.known_lt(b2):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -323,6 +323,8 @@
 def register_replacement_for(replaced_function, sandboxed_name=None):
     def wrap(func):
         from rpython.rtyper.extregistry import ExtRegistryEntry
+        # to support calling func directly
+        func._sandbox_external_name = sandboxed_name
         class ExtRegistry(ExtRegistryEntry):
             _about_ = replaced_function
             def compute_annotation(self):
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -1,6 +1,7 @@
 """Float constants"""
 
 import math, struct
+from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p, expm1
 
 from rpython.annotator.model import SomeString, SomeChar
 from rpython.rlib import objectmodel, unroll
@@ -184,104 +185,6 @@
 INFINITY = 1e200 * 1e200
 NAN = abs(INFINITY / INFINITY)    # bah, INF/INF gives us -NAN?
 
-try:
-    # Try to get math functions added in 2.6.
-    from math import isinf, isnan, copysign, acosh, asinh, atanh, log1p
-except ImportError:
-    @not_rpython
-    def isinf(x):
-        return x == INFINITY or x == -INFINITY
-
-    @not_rpython
-    def isnan(v):
-        return v != v
-
-    @not_rpython
-    def copysign(x, y):
-        """Return x with the sign of y"""
-        if x < 0.:
-            x = -x
-        if y > 0. or (y == 0. and math.atan2(y, -1.) > 0.):
-            return x
-        else:
-            return -x
-
-    _2_to_m28 = 3.7252902984619141E-09; # 2**-28
-    _2_to_p28 = 268435456.0; # 2**28
-    _ln2 = 6.93147180559945286227E-01
-
-    @not_rpython
-    def acosh(x):
-        if isnan(x):
-            return NAN
-        if x < 1.:
-            raise ValueError("math domain error")
-        if x >= _2_to_p28:
-            if isinf(x):
-                return x
-            else:
-                return math.log(x) + _ln2
-        if x == 1.:
-            return 0.
-        if x >= 2.:
-            t = x * x
-            return math.log(2. * x - 1. / (x + math.sqrt(t - 1.0)))
-        t = x - 1.0
-        return log1p(t + math.sqrt(2. * t + t * t))
-
-    @not_rpython
-    def asinh(x):
-        absx = abs(x)
-        if not isfinite(x):
-            return x
-        if absx < _2_to_m28:
-            return x
-        if absx > _2_to_p28:
-            w = math.log(absx) + _ln2
-        elif absx > 2.:
-            w = math.log(2. * absx + 1. / (math.sqrt(x * x + 1.) + absx))
-        else:
-            t = x * x
-            w = log1p(absx + t / (1. + math.sqrt(1. + t)))
-        return copysign(w, x)
-
-    @not_rpython
-    def atanh(x):
-        if isnan(x):
-            return x
-        absx = abs(x)
-        if absx >= 1.:
-            raise ValueError("math domain error")
-        if absx < _2_to_m28:
-            return x
-        if absx < .5:
-            t = absx + absx
-            t = .5 * log1p(t + t * absx / (1. - absx))
-        else:
-            t = .5 * log1p((absx + absx) / (1. - absx))
-        return copysign(t, x)
-
-    @not_rpython
-    def log1p(x):
-        if abs(x) < DBL_EPSILON // 2.:
-            return x
-        elif -.5 <= x <= 1.:
-            y = 1. + x
-            return math.log(y) - ((y - 1.) - x) / y
-        else:
-            return math.log(1. + x)
-
-try:
-    from math import expm1 # Added in Python 2.7.
-except ImportError:
-    @not_rpython
-    def expm1(x):
-        if abs(x) < .7:
-            u = math.exp(x)
-            if u == 1.:
-                return x
-            return (u - 1.) * x / math.log(u)
-        return math.exp(x) - 1.
 
 def log2(x):
     # Uses an algorithm that should:
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -65,6 +65,24 @@
     f.close()
     assert tail == ""
 
+def test_open_dup_rposix():
+    from rpython.rlib import rposix
+    def entry_point(argv):
+        fd = rposix.open("/tmp/foobar", os.O_RDONLY, 0777)
+        assert fd == 77
+        fd2 = rposix.dup(fd)
+        assert fd2 == 78
+        return 0
+
+    exe = compile(entry_point)
+    g, f = run_in_subprocess(exe)
+    expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
+    expect(f, g, "ll_os.ll_os_dup",  (77, True), 78)
+    g.close()
+    tail = f.read()
+    f.close()
+    assert tail == ""
+
 def test_read_write():
     def entry_point(argv):
         fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)
diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py
--- a/rpython/translator/translator.py
+++ b/rpython/translator/translator.py
@@ -67,7 +67,8 @@
         if self.annotator is not None:
             raise ValueError("we already have an annotator")
         from rpython.annotator.annrpython import RPythonAnnotator
-        self.annotator = RPythonAnnotator(self, policy=policy)
+        self.annotator = RPythonAnnotator(
+            self, policy=policy, keepgoing=self.config.translation.keepgoing)
         return self.annotator
 
     def buildrtyper(self):

From pypy.commits at gmail.com  Thu Nov 24 13:19:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 24 Nov 2016 10:19:59 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Reinstate the annotation hack that
 was removed in db07794f9b14
Message-ID: <58372f4f.c6bdc20a.29d75.a1bd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88646:3c5e062cf847
Date: 2016-11-24 18:18 +0000
http://bitbucket.org/pypy/pypy/changeset/3c5e062cf847/

Log:	Reinstate the annotation hack that was removed in db07794f9b14

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -510,7 +510,13 @@
         results = []
         for desc in pbc.descriptions:
             results.append(desc.pycall(whence, args, v_result, op))
-        s_result = unionof(*results)
+        # According to the fundamental annotation invariant, we should have
+        # unionof(*results).contains(v_result.annotation), but some
+        # specializers (e.g. argtype) don't respect that, so we make sure to
+        # return a generalisation of v_result.annotation.
+        s_result = v_result.annotation if v_result and v_result.annotation else s_ImpossibleValue
+        for result in results:
+            s_result = unionof(s_result, result)
         return s_result
 
     def emulate_pbc_call(self, unique_key, pbc, args_s, replace=[], callback=None):

From pypy.commits at gmail.com  Thu Nov 24 16:43:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 13:43:51 -0800 (PST)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <58375f17.c19d1c0a.22254.a0d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88647:f75cce6ff4e9
Date: 2016-11-24 22:42 +0100
http://bitbucket.org/pypy/pypy/changeset/f75cce6ff4e9/

Log:	translation fix

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1191,7 +1191,7 @@
     else:
         if condition:
             function(*args)
-conditional_call._always_inline_ = True
+conditional_call._always_inline_ = 'try'
 
 def _jit_conditional_call_value(value, function, *args):
     return value    # special-cased below
@@ -1233,7 +1233,7 @@
                 value = function(*args)
                 assert not isinstance(value, int)
         return value
-conditional_call_elidable._always_inline_ = True
+conditional_call_elidable._always_inline_ = 'try'
 
 class ConditionalCallEntry(ExtRegistryEntry):
     _about_ = _jit_conditional_call, _jit_conditional_call_value

From pypy.commits at gmail.com  Thu Nov 24 16:43:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Thu, 24 Nov 2016 13:43:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <58375f19.0d1a1c0a.c1daa.a2cc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88648:bf57a8b4e6fd
Date: 2016-11-24 22:42 +0100
http://bitbucket.org/pypy/pypy/changeset/bf57a8b4e6fd/

Log:	hg merge default

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1191,7 +1191,7 @@
     else:
         if condition:
             function(*args)
-conditional_call._always_inline_ = True
+conditional_call._always_inline_ = 'try'
 
 def _jit_conditional_call_value(value, function, *args):
     return value    # special-cased below
@@ -1233,7 +1233,7 @@
                 value = function(*args)
                 assert not isinstance(value, int)
         return value
-conditional_call_elidable._always_inline_ = True
+conditional_call_elidable._always_inline_ = 'try'
 
 class ConditionalCallEntry(ExtRegistryEntry):
     _about_ = _jit_conditional_call, _jit_conditional_call_value

From pypy.commits at gmail.com  Fri Nov 25 03:17:47 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 00:17:47 -0800 (PST)
Subject: [pypy-commit] pypy default: test fix
Message-ID: <5837f3ab.471ec20a.76ac6.7b93@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88651:9f86ca85efa9
Date: 2016-11-25 09:17 +0100
http://bitbucket.org/pypy/pypy/changeset/9f86ca85efa9/

Log:	test fix

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -562,6 +562,9 @@
     def op_jit_conditional_call(self, *args):
         raise NotImplementedError("should not be called while not jitted")
 
+    def op_jit_conditional_call_value(self, *args):
+        raise NotImplementedError("should not be called while not jitted")
+
     def op_get_exception_addr(self, *args):
         raise NotImplementedError
 

From pypy.commits at gmail.com  Fri Nov 25 03:19:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 00:19:08 -0800 (PST)
Subject: [pypy-commit] pypy default: document branch
Message-ID: <5837f3fc.973f1c0a.1ba45.3a6e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88652:2a459fd79347
Date: 2016-11-25 09:18 +0100
http://bitbucket.org/pypy/pypy/changeset/2a459fd79347/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,3 +20,8 @@
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt
+
+.. branch: conditional_call_value_4
+
+Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
+call this function" returning a result.

From pypy.commits at gmail.com  Fri Nov 25 03:24:31 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 00:24:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <5837f53f.96a61c0a.e4f42.44e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88653:87cc1c9f050a
Date: 2016-11-25 09:23 +0100
http://bitbucket.org/pypy/pypy/changeset/87cc1c9f050a/

Log:	fix

diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -123,5 +123,5 @@
     # and returns a "char *" inside this PyStringObject.
     if not isinstance(w_mod, Module):
         raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
-    from pypy.module.cpyext.bytesobject import PyString_AsString
-    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
+    from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
+    return _PyUnicode_AsString(space, as_pyobj(space, w_mod.w_name))

From pypy.commits at gmail.com  Fri Nov 25 05:37:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 02:37:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Copy-paste customize_compiler()
 unmodified from sysconfig_cpython, i.e.
Message-ID: <5838147f.ca57c20a.49dfd.a9f7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88654:7c6299aa186e
Date: 2016-11-25 11:37 +0100
http://bitbucket.org/pypy/pypy/changeset/7c6299aa186e/

Log:	Copy-paste customize_compiler() unmodified from sysconfig_cpython,
	i.e. handling all environment variables the same way

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,17 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
     g['VERSION'] = get_python_version()
 
     global _config_vars
@@ -119,30 +124,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (

From pypy.commits at gmail.com  Fri Nov 25 05:39:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 25 Nov 2016 02:39:06 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: remove the remaining relevant
 wraps in micronumpy (hopefully)
Message-ID: <583814ca.c8111c0a.5fdd4.7703@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88655:97f865748606
Date: 2016-11-25 11:20 +0100
http://bitbucket.org/pypy/pypy/changeset/97f865748606/

Log:	remove the remaining relevant wraps in micronumpy (hopefully)

diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -18,7 +18,7 @@
     def call_method(self, space, path, name, args):
         w_method = getattr(self, 'w_' + name)
         if w_method is None:
-            w_method = space.appexec([space.wrap(path), space.wrap(name)],
+            w_method = space.appexec([space.newtext(path), space.newtext(name)],
                 "(path, name): return getattr(__import__(path, fromlist=[name]), name)")
             setattr(self, 'w_' + name, w_method)
         return space.call_args(w_method, args)
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -613,7 +613,8 @@
 
 class W_CharacterBox(W_FlexibleBox):
     def convert_to(self, space, dtype):
-        return dtype.coerce(space, space.wrap(self.raw_str()))
+        # XXX should be newbytes?
+        return dtype.coerce(space, space.newtext(self.raw_str()))
 
     def descr_len(self, space):
         return space.len(self.item(space))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -409,9 +409,12 @@
     def newdict(self, module=True):
         return DictObject({})
 
+    @specialize.argtype(1)
     def newint(self, i):
         if isinstance(i, IntObject):
             return i
+        if isinstance(i, base_int):
+            return LongObject(i)
         return IntObject(i)
 
     def setitem(self, obj, index, value):
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -468,7 +468,7 @@
                 val = dtype.itemtype.default_fromstring(space)
             else:
                 try:
-                    val = dtype.coerce(space, space.wrap(piece))
+                    val = dtype.coerce(space, space.newtext(piece))
                 except OperationError as e:
                     if not e.match(space, space.w_ValueError):
                         raise
@@ -476,7 +476,7 @@
                     while not gotit and len(piece) > 0:
                         piece = piece[:-1]
                         try:
-                            val = dtype.coerce(space, space.wrap(piece))
+                            val = dtype.coerce(space, space.newtext(piece))
                             gotit = True
                         except OperationError as e:
                             if not e.match(space, space.w_ValueError):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1271,7 +1271,7 @@
                             op_name, self.get_dtype().get_name())
             shape = self.get_shape()
             if space.is_none(w_axis) or len(shape) <= 1:
-                return space.wrap(getattr(loop, op_name_flat)(self))
+                return space.newint(getattr(loop, op_name_flat)(self))
             else:
                 axis = space.int_w(w_axis)
                 assert axis >= 0
@@ -1359,7 +1359,7 @@
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
-            [space.newint(0)]), space.wrap("b")])
+            [space.newint(0)]), space.newtext("b")])
 
         builder = StringBuilder()
         if self.get_dtype().is_object():
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -462,7 +462,7 @@
     signed = True
 
     def to_builtin_type(self, space, box):
-        return space.wrap(self.for_computation(self.unbox(box)))
+        return space.newint(self.for_computation(self.unbox(box)))
 
     def _base_coerce(self, space, w_item):
         if w_item is None:
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1206,7 +1206,7 @@
     nargs = interp_attrproperty("nargs", cls=W_Ufunc,
         wrapfn="newint"),
     signature = interp_attrproperty("signature", cls=W_Ufunc,
-        wrapfn="newtext"),
+        wrapfn="newtext_or_none"),
 
     reduce = interp2app(W_Ufunc.descr_reduce),
     outer = interp2app(W_Ufunc.descr_outer),

From pypy.commits at gmail.com  Fri Nov 25 05:46:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 02:46:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Copy-paste customize_compiler()
 unmodified from sysconfig_cpython, 
Message-ID: <58381680.6a5cc20a.a0487.b08e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88657:50b42f56b6c5
Date: 2016-11-25 11:45 +0100
http://bitbucket.org/pypy/pypy/changeset/50b42f56b6c5/

Log:	Copy-paste customize_compiler() unmodified from sysconfig_cpython,
	i.e. handling all environment variables the same way

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 
 from distutils.errors import DistutilsPlatformError
 
@@ -61,11 +60,19 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['SHLIB_SUFFIX'] = g['SO']
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -76,7 +83,7 @@
     g = {}
     g['EXE'] = ".exe"
     g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = g['SO'].rsplit('.')[0]   # xxx?
 
     global _config_vars
     _config_vars = g
@@ -118,26 +125,78 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+        (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = shlib_suffix
 
 
 from .sysconfig_cpython import (

From pypy.commits at gmail.com  Fri Nov 25 05:46:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 02:46:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5838167f.113cc20a.8d5cb.a144@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88656:4d85babd12e1
Date: 2016-11-25 11:45 +0100
http://bitbucket.org/pypy/pypy/changeset/4d85babd12e1/

Log:	hg merge default

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,17 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
     g['VERSION'] = get_python_version()
 
     global _config_vars
@@ -119,30 +124,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -29,3 +29,8 @@
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt
+
+.. branch: conditional_call_value_4
+
+Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
+call this function" returning a result.
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -562,6 +562,9 @@
     def op_jit_conditional_call(self, *args):
         raise NotImplementedError("should not be called while not jitted")
 
+    def op_jit_conditional_call_value(self, *args):
+        raise NotImplementedError("should not be called while not jitted")
+
     def op_get_exception_addr(self, *args):
         raise NotImplementedError
 

From pypy.commits at gmail.com  Fri Nov 25 05:49:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 02:49:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Missing import
Message-ID: <58381726.4dd41c0a.128c7.7de1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88658:b33f93f52082
Date: 2016-11-25 11:48 +0100
http://bitbucket.org/pypy/pypy/changeset/b33f93f52082/

Log:	Missing import

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -12,6 +12,7 @@
 
 import sys
 import os
+import imp
 
 from distutils.errors import DistutilsPlatformError
 

From pypy.commits at gmail.com  Fri Nov 25 08:30:21 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 25 Nov 2016 05:30:21 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: more various wraps
Message-ID: <58383ced.c6bdc20a.29d75.0a62@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88659:000765df2300
Date: 2016-11-25 14:29 +0100
http://bitbucket.org/pypy/pypy/changeset/000765df2300/

Log:	more various wraps

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -495,13 +495,13 @@
         else:
             name = importname
 
-        mod = Module(self, self.wrap(name))
+        mod = Module(self, self.newtext(name))
         mod.install()
 
         return name
 
     def getbuiltinmodule(self, name, force_init=False, reuse=True):
-        w_name = self.wrap(name)
+        w_name = self.newtext(name)
         w_modules = self.sys.get('modules')
         if not force_init:
             assert reuse
@@ -527,7 +527,7 @@
             if not reuse and w_mod.startup_called:
                 # create a copy of the module.  (see issue1514) eventlet
                 # patcher relies on this behaviour.
-                w_mod2 = self.wrap(Module(self, w_name))
+                w_mod2 = Module(self, w_name)
                 self.setitem(w_modules, w_name, w_mod2)
                 w_mod.getdict(self)  # unlazy w_initialdict
                 self.call_method(w_mod2.getdict(self), 'update',
@@ -583,26 +583,26 @@
         "NOT_RPYTHON: only for initializing the space."
 
         from pypy.module.exceptions import Module
-        w_name = self.wrap('exceptions')
+        w_name = self.newtext('exceptions')
         self.exceptions_module = Module(self, w_name)
         self.exceptions_module.install()
 
         from pypy.module.sys import Module
-        w_name = self.wrap('sys')
+        w_name = self.newtext('sys')
         self.sys = Module(self, w_name)
         self.sys.install()
 
         from pypy.module.imp import Module
-        w_name = self.wrap('imp')
+        w_name = self.newtext('imp')
         mod = Module(self, w_name)
         mod.install()
 
         from pypy.module.__builtin__ import Module
-        w_name = self.wrap('__builtin__')
+        w_name = self.newtext('__builtin__')
         self.builtin = Module(self, w_name)
         w_builtin = self.wrap(self.builtin)
         w_builtin.install()
-        self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
+        self.setitem(self.builtin.w_dict, self.newtext('__builtins__'), w_builtin)
 
         bootstrap_modules = set(('sys', 'imp', '__builtin__', 'exceptions'))
         installed_builtin_modules = list(bootstrap_modules)
@@ -613,7 +613,7 @@
         types_w = (self.get_builtin_types().items() +
                    exception_types_w.items())
         for name, w_type in types_w:
-            self.setitem(self.builtin.w_dict, self.wrap(name), w_type)
+            self.setitem(self.builtin.w_dict, self.newtext(name), w_type)
 
         # install mixed modules
         for mixedname in self.get_builtinmodule_to_install():
@@ -625,7 +625,7 @@
             [self.wrap(fn) for fn in installed_builtin_modules])
 
         # force this value into the dict without unlazyfying everything
-        self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'),
+        self.setitem(self.sys.w_dict, self.newtext('builtin_module_names'),
                      w_builtin_module_names)
 
     def get_builtin_types(self):
@@ -744,7 +744,7 @@
     # may also override specific functions for performance.
 
     def not_(self, w_obj):
-        return self.wrap(not self.is_true(w_obj))
+        return self.newbool(not self.is_true(w_obj))
 
     def eq_w(self, w_obj1, w_obj2):
         """Implements equality with the double check 'x is y or x == y'."""
@@ -770,7 +770,7 @@
         w_result = w_obj.immutable_unique_id(self)
         if w_result is None:
             # in the common case, returns an unsigned value
-            w_result = self.wrap(r_uint(compute_unique_id(w_obj)))
+            w_result = self.newint(r_uint(compute_unique_id(w_obj)))
         return w_result
 
     def hash_w(self, w_obj):
@@ -1056,13 +1056,13 @@
         return self.newlist([self.newbytes(s) for s in list_s])
 
     def newlist_unicode(self, list_u):
-        return self.newlist([self.wrap(u) for u in list_u])
+        return self.newlist([self.newunicode(u) for u in list_u])
 
     def newlist_int(self, list_i):
-        return self.newlist([self.wrap(i) for i in list_i])
+        return self.newlist([self.newint(i) for i in list_i])
 
     def newlist_float(self, list_f):
-        return self.newlist([self.wrap(f) for f in list_f])
+        return self.newlist([self.newfloat(f) for f in list_f])
 
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -20,7 +20,7 @@
         char = __builtin__.chr(space.int_w(w_ascii))
     except ValueError:  # chr(out-of-range)
         raise oefmt(space.w_ValueError, "character code not in range(256)")
-    return space.wrap(char) # YYY
+    return space.newtext(char)
 
 @unwrap_spec(code=int)
 def unichr(space, code):
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -16,7 +16,9 @@
     def startup(self, space):
         space.fromcache(State).startup(space)
         method = pypy.module.cpyext.typeobject.get_new_method_def(space)
-        w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
+        # the w_self argument here is a dummy, the only thing done with w_obj
+        # is call space.type on it
+        w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.w_None)
         space.appexec([space.type(w_obj)], """(methodtype):
             from pickle import Pickler 
             Pickler.dispatch[methodtype] = Pickler.save_global
diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py
--- a/pypy/module/cpyext/ndarrayobject.py
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -264,7 +264,7 @@
     w_dtypes = space.newlist(dtypes_w)
     w_doc = rffi.charp2str(doc)
     w_name = rffi.charp2str(name)
-    w_identity = space.wrap(identity)
+    w_identity = space.newint(identity)
     ufunc_generic = ufuncs.frompyfunc(space, w_funcs, nin, nout, w_dtypes,
                  w_signature, w_identity, w_name, w_doc, stack_inputs=True)
     return ufunc_generic
diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py
--- a/pypy/module/cpyext/structmember.py
+++ b/pypy/module/cpyext/structmember.py
@@ -67,7 +67,7 @@
         w_result = PyString_FromString(space, result)
     elif member_type == T_CHAR:
         result = rffi.cast(rffi.CCHARP, addr)
-        w_result = space.wrap(result[0])
+        w_result = space.newtext(result[0])
     elif member_type == T_OBJECT:
         obj_ptr = rffi.cast(PyObjectP, addr)
         if obj_ptr[0]:
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
--- a/pypy/module/sys/currentframes.py
+++ b/pypy/module/sys/currentframes.py
@@ -64,7 +64,7 @@
         w_topframe = space.w_None
         w_prevframe = None
         for f in frames:
-            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            w_nextframe = space.call_function(w_fake_frame, space.wrap_none(f))
             if w_prevframe is None:
                 w_topframe = w_nextframe
             else:

From pypy.commits at gmail.com  Fri Nov 25 08:30:59 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:30:59 -0800 (PST)
Subject: [pypy-commit] pypy desc-specialize: FunctionDesc specializers
 always return graphs now
Message-ID: <58383d13.c4251c0a.ab6d3.bbf7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: desc-specialize
Changeset: r88661:234b2c30c81b
Date: 2016-11-25 00:24 +0000
http://bitbucket.org/pypy/pypy/changeset/234b2c30c81b/

Log:	FunctionDesc specializers always return graphs now

diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -282,23 +282,22 @@
 
     def pycall(self, whence, args, s_previous_result, op=None):
         inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
         # Some specializations may break the invariant of returning
         # annotations that are always more general than the previous time.
         # We restore it here:

From pypy.commits at gmail.com  Fri Nov 25 08:30:57 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:30:57 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Create annotator.get_op()
Message-ID: <58383d11.448e1c0a.79cea.bd37@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88660:01e41e1ab1a1
Date: 2016-11-24 20:49 +0000
http://bitbucket.org/pypy/pypy/changeset/01e41e1ab1a1/

Log:	Create annotator.get_op()

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -280,9 +280,12 @@
         tag = parent_block, parent_index
         self.translator.update_call_graph(parent_graph, graph, tag)
 
+    def get_op(self, position_key):
+        _, block, index = position_key
+        return block.operations[index]
+
     def get_result_var(self, position_key):
-        _, block, index = position_key
-        op = block.operations[index]
+        op = self.get_op(position_key)
         v_result = op.result
         self.var_def[v_result] = position_key
         return v_result
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -499,10 +499,8 @@
             whence = emulated
 
         if whence is not None:
-            fn, block, i = whence
-            op = block.operations[i]
-            v_result = op.result
-            self.annotator.var_def[v_result] = whence
+            op = self.annotator.get_op(whence)
+            v_result = self.annotator.get_result_var(whence)
         else:
             op = None
             v_result = None

From pypy.commits at gmail.com  Fri Nov 25 08:31:04 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:04 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Attach notifications to variables
 rather than blocks
Message-ID: <58383d18.849c1c0a.38a6c.b956@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88664:4b2aca8c3977
Date: 2016-11-25 06:55 +0000
http://bitbucket.org/pypy/pypy/changeset/4b2aca8c3977/

Log:	Attach notifications to variables rather than blocks

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -294,6 +294,10 @@
         self.var_def[v_result] = position_key
         return v_result
 
+    def add_notification(self, v_src, v_target):
+        successors = self.notify.setdefault(v_src, set())
+        successors.add(v_target)
+
     def update_var(self, v):
         position_key = self.var_def[v]
         self.reflowfromposition(position_key)
@@ -449,6 +453,10 @@
         return repr(graph) + blk + opid
 
     def flowin(self, graph, block):
+        for v in block.inputargs:
+            if v in self.notify:
+                for v2 in self.notify[v]:
+                    self.update_var(v2)
         try:
             i = 0
             while i < len(block.operations):
@@ -534,10 +542,6 @@
                 constraints = knowntypedata.get(link.exitcase, {})
                 self.follow_link(graph, link, constraints)
 
-        if block in self.notify:
-            for v in self.notify[block]:
-                self.update_var(v)
-
     def follow_link(self, graph, link, constraints):
         assert not (isinstance(link.exitcase, (types.ClassType, type)) and
                 issubclass(link.exitcase, BaseException))
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -293,10 +293,7 @@
         if whence is not None:
             annotator.record_call(graph, whence)
         if v_result is not None:
-            # annotator.notify[graph.returnblock] is a set of variables to update
-            # whenever the return block of this graph has been analysed.
-            returnvars = annotator.notify.setdefault(graph.returnblock, set())
-            returnvars.add(v_result)
+            annotator.add_notification(graph.getreturnvar(), v_result)
 
         # generalize the function's input arguments
         annotator.addpendinggraph(graph, inputcells)

From pypy.commits at gmail.com  Fri Nov 25 08:31:01 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:01 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: hg merge desc-specialize
Message-ID: <58383d15.4f831c0a.80d3.b73b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88662:85bd42cecd01
Date: 2016-11-25 04:22 +0000
http://bitbucket.org/pypy/pypy/changeset/85bd42cecd01/

Log:	hg merge desc-specialize

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -439,6 +439,7 @@
                                           space.wrap(msg))
         return OperationError(exc, w_error)
 
+ at specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
                   w_exception_class=None):
     assert isinstance(e, OSError)
@@ -466,8 +467,8 @@
         w_error = space.call_function(exc, space.wrap(errno),
                                       space.wrap(msg))
     return OperationError(exc, w_error)
-wrap_oserror2._annspecialcase_ = 'specialize:arg(3)'
 
+ at specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
                  w_exception_class=None):
     if filename is not None:
@@ -478,7 +479,6 @@
         return wrap_oserror2(space, e, None,
                              exception_name=exception_name,
                              w_exception_class=w_exception_class)
-wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -4,6 +4,7 @@
 sys.path.insert(0, str(ROOT))
 
 import time
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
 from rpython.rlib.objectmodel import specialize, dont_inline
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -124,13 +124,13 @@
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         if can_be_None and w_obj is None:
             return None
         if not isinstance(w_obj, RequiredClass):
             raise TypeError
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def getarg_w(self, code, w_obj):    # for retrieving buffers
         return FakeBuffer(w_obj)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1614,9 +1614,8 @@
     miniglobals = {'__name__':    __name__, # for module name propagation
                    }
     exec source.compile() in miniglobals
-    call_external_function = miniglobals['cpy_call_external']
+    call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
     call_external_function._dont_inline_ = True
-    call_external_function._annspecialcase_ = 'specialize:ll'
     call_external_function._gctransformer_hint_close_stack_ = True
     # don't inline, as a hack to guarantee that no GC pointer is alive
     # anywhere in call_external_function
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -3,6 +3,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -202,6 +202,7 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):
@@ -215,7 +216,6 @@
                 return w_some_obj()
             self._wrap_not_rpython(x)
         return w_some_obj()
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
@@ -305,10 +305,10 @@
         is_root(w_complex)
         return 1.1, 2.2
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     def decode_index(self, w_index_or_slice, seqlength):
         is_root(w_index_or_slice)
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -9,9 +9,9 @@
 
 def make_checker():
     check = []
+    @specialize.memo()
     def see():
         check.append(True)
-    see._annspecialcase_ = 'specialize:memo'
     return see, check
 
 def test_wrap_interp2app():
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -2,6 +2,7 @@
 import sys
 
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import INT_MAX
 from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -2,6 +2,7 @@
 
 import types
 from collections import defaultdict
+from contextlib import contextmanager
 
 from rpython.tool.ansi_print import AnsiLogger
 from rpython.tool.pairtype import pair
@@ -84,22 +85,17 @@
         annmodel.TLS.check_str_without_nul = (
             self.translator.config.translation.check_str_without_nul)
 
-        flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
+        with self.using_policy(policy):
+            flowgraph, inputs_s = self.get_call_parameters(function, args_s)
 
         if main_entry_point:
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
-    def get_call_parameters(self, function, args_s, policy):
-        desc = self.bookkeeper.getdesc(function)
-        prevpolicy = self.policy
-        self.policy = policy
-        self.bookkeeper.enter(None)
-        try:
+    def get_call_parameters(self, function, args_s):
+        with self.bookkeeper.at_position(None):
+            desc = self.bookkeeper.getdesc(function)
             return desc.get_call_parameters(args_s)
-        finally:
-            self.bookkeeper.leave()
-            self.policy = prevpolicy
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
@@ -108,21 +104,29 @@
             # XXX hack
             annmodel.TLS.check_str_without_nul = (
                 self.translator.config.translation.check_str_without_nul)
-        graph, inputcells = self.get_call_parameters(function, args_s, policy)
-        self.build_graph_types(graph, inputcells, complete_now=False)
-        self.complete_helpers(policy)
+        with self.using_policy(policy):
+            graph, inputcells = self.get_call_parameters(function, args_s)
+            self.build_graph_types(graph, inputcells, complete_now=False)
+            self.complete_helpers()
         return graph
 
-    def complete_helpers(self, policy):
-        saved = self.policy, self.added_blocks
-        self.policy = policy
+    def complete_helpers(self):
+        saved = self.added_blocks
+        self.added_blocks = {}
         try:
-            self.added_blocks = {}
             self.complete()
             # invoke annotation simplifications for the new blocks
             self.simplify(block_subset=self.added_blocks)
         finally:
-            self.policy, self.added_blocks = saved
+            self.added_blocks = saved
+
+    @contextmanager
+    def using_policy(self, policy):
+        """A context manager that temporarily replaces the annotator policy"""
+        old_policy = self.policy
+        self.policy = policy
+        yield
+        self.policy = old_policy
 
     def build_graph_types(self, flowgraph, inputcells, complete_now=True):
         checkgraph(flowgraph)
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -9,6 +9,7 @@
 from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
+from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.model import (
     SomeOrderedDict, SomeString, SomeChar, SomeFloat, unionof, SomeInstance,
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
@@ -21,6 +22,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
+from rpython.annotator.specialize import memo
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
@@ -358,7 +360,7 @@
             return self.descs[obj_key]
         except KeyError:
             if isinstance(pyobj, types.FunctionType):
-                result = description.FunctionDesc(self, pyobj)
+                result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
                     raise Exception("ClassDesc for object not supported")
@@ -403,6 +405,23 @@
             self.descs[obj_key] = result
             return result
 
+    def newfuncdesc(self, pyfunc):
+        name = pyfunc.__name__
+        if hasattr(pyfunc, '_generator_next_method_of_'):
+            from rpython.flowspace.argument import Signature
+            signature = Signature(['entry'])     # haaaaaack
+            defaults = ()
+        else:
+            signature = cpython_code_signature(pyfunc.func_code)
+            defaults = pyfunc.func_defaults
+        # get the specializer based on the tag of the 'pyobj'
+        # (if any), according to the current policy
+        tag = getattr(pyfunc, '_annspecialcase_', None)
+        specializer = self.annotator.policy.get_specializer(tag)
+        if specializer is memo:
+            return description.MemoDesc(self, pyfunc, name, signature, defaults, specializer)
+        return description.FunctionDesc(self, pyfunc, name, signature, defaults, specializer)
+
     def getfrozen(self, pyobj):
         return description.FrozenDesc(self, pyobj)
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -608,7 +608,7 @@
             if mixin:
                 # make a new copy of the FunctionDesc for this class,
                 # but don't specialize further for all subclasses
-                funcdesc = FunctionDesc(self.bookkeeper, value)
+                funcdesc = self.bookkeeper.newfuncdesc(value)
                 self.classdict[name] = funcdesc
                 return
             # NB. if value is, say, AssertionError.__init__, then we
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -3,11 +3,10 @@
 from rpython.annotator.signature import (
     enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import FunctionGraph
-from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, s_ImpossibleValue
+from rpython.annotator.model import AnnotatorError, s_ImpossibleValue, unionof
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -117,7 +116,6 @@
         self.s_value = s_ImpossibleValue    # union of possible values
 
     def update(self, other):
-        from rpython.annotator.model import unionof
         self.descs.update(other.descs)
         self.read_locations.update(other.read_locations)
         self.s_value = unionof(self.s_value, other.s_value)
@@ -192,24 +190,12 @@
 class FunctionDesc(Desc):
     knowntype = types.FunctionType
 
-    def __init__(self, bookkeeper, pyobj=None,
-                 name=None, signature=None, defaults=None,
+    def __init__(self, bookkeeper, pyobj, name, signature, defaults,
                  specializer=None):
         super(FunctionDesc, self).__init__(bookkeeper, pyobj)
-        if name is None:
-            name = pyobj.func_name
-        if signature is None:
-            if hasattr(pyobj, '_generator_next_method_of_'):
-                from rpython.flowspace.argument import Signature
-                signature = Signature(['entry'])     # haaaaaack
-                defaults = ()
-            else:
-                signature = cpython_code_signature(pyobj.func_code)
-        if defaults is None:
-            defaults = pyobj.func_defaults
         self.name = name
         self.signature = signature
-        self.defaults = defaults or ()
+        self.defaults = defaults if defaults is not None else ()
         # 'specializer' is a function with the following signature:
         #      specializer(funcdesc, args_s) => graph
         #                                 or => s_result (overridden/memo cases)
@@ -288,12 +274,39 @@
                 getattr(self.bookkeeper, "position_key", None) is not None):
             _, block, i = self.bookkeeper.position_key
             op = block.operations[i]
-        if self.specializer is None:
-            # get the specializer based on the tag of the 'pyobj'
-            # (if any), according to the current policy
-            tag = getattr(self.pyobj, '_annspecialcase_', None)
-            policy = self.bookkeeper.annotator.policy
-            self.specializer = policy.get_specializer(tag)
+        self.normalize_args(inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
+
+    def pycall(self, whence, args, v_result, op=None):
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells, v_result)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
+        return result
+
+    def normalize_args(self, inputs_s):
+        """
+        Canonicalize argument annotations into the exact parameter
+        annotations of a specific specialized graph.
+
+        Note: this method has no return value but mutates its argument instead.
+        """
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
@@ -304,34 +317,9 @@
                 from rpython.annotator.signature import Sig
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
-            enforceargs(self, inputcells)  # can modify inputcells in-place
+            enforceargs(self, inputs_s)  # can modify inputs_s in-place
         if signature:
-            enforce_signature_args(self, signature[0], inputcells)  # mutates inputcells
-        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
-            return self.specializer(self, inputcells, op)
-        else:
-            return self.specializer(self, inputcells)
-
-    def pycall(self, whence, args, v_result, op=None):
-        inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells, v_result)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
-        return result
+            enforce_signature_args(self, signature[0], inputs_s)  # mutates inputs_s
 
     def get_graph(self, args, op):
         inputs_s = self.parse_arguments(args)
@@ -400,6 +388,15 @@
 
             return s_sigs
 
+class MemoDesc(FunctionDesc):
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        s_result = self.specialize(inputcells, op)
+        if isinstance(s_result, FunctionGraph):
+            s_result = s_result.getreturnvar().annotation
+        return s_result
+
+
 class MethodDesc(Desc):
     knowntype = types.MethodType
 
diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py
--- a/rpython/annotator/specialize.py
+++ b/rpython/annotator/specialize.py
@@ -3,11 +3,13 @@
 
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import Block, Link, Variable, SpaceOperation
+from rpython.flowspace.model import Block, Link, Variable
 from rpython.flowspace.model import checkgraph
 from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.flowspace.argument import Signature
+from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
+from rpython.annotator.model import unionof
 
 def flatten_star_args(funcdesc, args_s):
     argnames, vararg, kwarg = funcdesc.signature
@@ -127,7 +129,6 @@
     def finish(self):
         if self.do_not_process:
             return
-        from rpython.annotator.model import unionof
         assert self.graph is None, "MemoTable already finished"
         # list of which argument positions can take more than one value
         example_args, example_value = self.table.iteritems().next()
@@ -246,34 +247,36 @@
             args_s.append(unionof(*values_s))
         annotator.addpendinggraph(self.graph, args_s)
 
+def all_values(s):
+    """Return the exhaustive list of possible values matching annotation `s`.
 
-def memo(funcdesc, arglist_s):
-    from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
-    from rpython.annotator.model import unionof
+    Raises `AnnotatorError` if no such (reasonably small) finite list exists.
+    """
+    if s.is_constant():
+        return [s.const]
+    elif isinstance(s, SomePBC):
+        values = []
+        assert not s.can_be_None, "memo call: cannot mix None and PBCs"
+        for desc in s.descriptions:
+            if desc.pyobj is None:
+                raise annmodel.AnnotatorError(
+                    "memo call with a class or PBC that has no "
+                    "corresponding Python object (%r)" % (desc,))
+            values.append(desc.pyobj)
+        return values
+    elif isinstance(s, SomeImpossibleValue):
+        return []
+    elif isinstance(s, SomeBool):
+        return [False, True]
+    else:
+        raise annmodel.AnnotatorError("memo call: argument must be a class "
+                                        "or a frozen PBC, got %r" % (s,))
+
+def memo(funcdesc, args_s):
     # call the function now, and collect possible results
-    argvalues = []
-    for s in arglist_s:
-        if s.is_constant():
-            values = [s.const]
-        elif isinstance(s, SomePBC):
-            values = []
-            assert not s.can_be_None, "memo call: cannot mix None and PBCs"
-            for desc in s.descriptions:
-                if desc.pyobj is None:
-                    raise annmodel.AnnotatorError(
-                        "memo call with a class or PBC that has no "
-                        "corresponding Python object (%r)" % (desc,))
-                values.append(desc.pyobj)
-        elif isinstance(s, SomeImpossibleValue):
-            return s    # we will probably get more possible args later
-        elif isinstance(s, SomeBool):
-            values = [False, True]
-        else:
-            raise annmodel.AnnotatorError("memo call: argument must be a class "
-                                          "or a frozen PBC, got %r" % (s,))
-        argvalues.append(values)
+
     # the list of all possible tuples of arguments to give to the memo function
-    possiblevalues = cartesian_product(argvalues)
+    possiblevalues = cartesian_product([all_values(s_arg) for s_arg in args_s])
 
     # a MemoTable factory -- one MemoTable per family of arguments that can
     # be called together, merged via a UnionFind.
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -138,11 +138,12 @@
         # get the graph of the mix-level helper ll_function and prepare it for
         # being annotated.  Annotation and RTyping should be done in a single shot
         # at the end with finish().
-        graph, args_s = self.rtyper.annotator.get_call_parameters(
-            ll_function, args_s, policy = self.policy)
+        ann = self.rtyper.annotator
+        with ann.using_policy(self.policy):
+            graph, args_s = ann.get_call_parameters(ll_function, args_s)
         for v_arg, s_arg in zip(graph.getargs(), args_s):
-            self.rtyper.annotator.setbinding(v_arg, s_arg)
-        self.rtyper.annotator.setbinding(graph.getreturnvar(), s_result)
+            ann.setbinding(v_arg, s_arg)
+        ann.setbinding(graph.getreturnvar(), s_result)
         #self.rtyper.annotator.annotated[graph.returnblock] = graph
         self.pending.append((ll_function, graph, args_s, s_result))
         return graph
@@ -224,16 +225,17 @@
         bk = ann.bookkeeper
         translator = ann.translator
         original_graph_count = len(translator.graphs)
-        for ll_function, graph, args_s, s_result in self.pending:
-            # mark the return block as already annotated, because the return var
-            # annotation was forced in getgraph() above.  This prevents temporary
-            # less general values reaching the return block from crashing the
-            # annotator (on the assert-that-new-binding-is-not-less-general).
-            ann.annotated[graph.returnblock] = graph
-            s_function = bk.immutablevalue(ll_function)
-            bk.emulate_pbc_call(graph, s_function, args_s)
-            self.newgraphs.add(graph)
-        ann.complete_helpers(self.policy)
+        with ann.using_policy(self.policy):
+            for ll_function, graph, args_s, s_result in self.pending:
+                # mark the return block as already annotated, because the return var
+                # annotation was forced in getgraph() above.  This prevents temporary
+                # less general values reaching the return block from crashing the
+                # annotator (on the assert-that-new-binding-is-not-less-general).
+                ann.annotated[graph.returnblock] = graph
+                s_function = bk.immutablevalue(ll_function)
+                bk.emulate_pbc_call(graph, s_function, args_s)
+                self.newgraphs.add(graph)
+            ann.complete_helpers()
         for ll_function, graph, args_s, s_result in self.pending:
             s_real_result = ann.binding(graph.getreturnvar())
             if s_real_result != s_result:
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -384,7 +384,6 @@
 def _sizeof_none(TYPE):
     assert not TYPE._is_varsize()
     return ItemOffset(TYPE)
-_sizeof_none._annspecialcase_ = 'specialize:memo'
 
 @specialize.memo()
 def _internal_array_field(TYPE):
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -362,9 +362,9 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         funcdesc, = s_pbc.descriptions
-        args = simple_args(args_s)
         with bk.at_position(None):
-            graph = funcdesc.get_graph(args, op)
+            argspec = simple_args(args_s)
+            graph = funcdesc.get_graph(argspec, op)
         llfn = self.rtyper.getcallable(graph)
         return inputconst(typeOf(llfn), llfn)
 
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -846,28 +846,29 @@
         rtyper = self.rtyper
         args_s = []
         newargs_v = []
-        for v in args_v:
-            if v.concretetype is Void:
-                s_value = rtyper.annotation(v)
-                if s_value is None:
-                    s_value = annmodel.s_None
-                if not s_value.is_constant():
-                    raise TyperError("non-constant variable of type Void")
-                if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
-                    raise TyperError("non-PBC Void argument: %r", (s_value,))
-                args_s.append(s_value)
-            else:
-                args_s.append(lltype_to_annotation(v.concretetype))
-            newargs_v.append(v)
+        with rtyper.annotator.using_policy(rtyper.lowlevel_ann_policy):
+            for v in args_v:
+                if v.concretetype is Void:
+                    s_value = rtyper.annotation(v)
+                    if s_value is None:
+                        s_value = annmodel.s_None
+                    if not s_value.is_constant():
+                        raise TyperError("non-constant variable of type Void")
+                    if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
+                        raise TyperError("non-PBC Void argument: %r", (s_value,))
+                    args_s.append(s_value)
+                else:
+                    args_s.append(lltype_to_annotation(v.concretetype))
+                newargs_v.append(v)
 
-        self.rtyper.call_all_setups()  # compute ForwardReferences now
+            self.rtyper.call_all_setups()  # compute ForwardReferences now
 
-        # hack for bound methods
-        if hasattr(ll_function, 'im_func'):
-            bk = rtyper.annotator.bookkeeper
-            args_s.insert(0, bk.immutablevalue(ll_function.im_self))
-            newargs_v.insert(0, inputconst(Void, ll_function.im_self))
-            ll_function = ll_function.im_func
+            # hack for bound methods
+            if hasattr(ll_function, 'im_func'):
+                bk = rtyper.annotator.bookkeeper
+                args_s.insert(0, bk.immutablevalue(ll_function.im_self))
+                newargs_v.insert(0, inputconst(Void, ll_function.im_self))
+                ll_function = ll_function.im_func
 
         graph = annotate_lowlevel_helper(rtyper.annotator, ll_function, args_s,
                                          rtyper.lowlevel_ann_policy)

From pypy.commits at gmail.com  Fri Nov 25 08:31:03 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:03 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Inline ann.recursivecall() into its
 only caller
Message-ID: <58383d17.e576c20a.d6e2c.e4f4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88663:ce4e321b09fa
Date: 2016-11-25 05:48 +0000
http://bitbucket.org/pypy/pypy/changeset/ce4e321b09fa/

Log:	Inline ann.recursivecall() into its only caller

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -294,28 +294,6 @@
         self.var_def[v_result] = position_key
         return v_result
 
-
-    def recursivecall(self, graph, whence, inputcells, v_result):
-        if whence is not None:
-            self.record_call(graph, whence)
-        if v_result is not None:
-            # self.notify[graph.returnblock] is a set of variables to update
-            # whenever the return block of this graph has been analysed.
-            returnvars = self.notify.setdefault(graph.returnblock, set())
-            returnvars.add(v_result)
-
-        # generalize the function's input arguments
-        self.addpendingblock(graph, graph.startblock, inputcells)
-
-        # get the (current) return value
-        v = graph.getreturnvar()
-        try:
-            return self.binding(v)
-        except KeyError:
-            # the function didn't reach any return statement so far.
-            # (some functions actually never do, they always raise exceptions)
-            return s_ImpossibleValue
-
     def update_var(self, v):
         position_key = self.var_def[v]
         self.reflowfromposition(position_key)
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -290,7 +290,20 @@
         new_args = args.unmatch_signature(self.signature, inputcells)
         inputcells = self.parse_arguments(new_args, graph)
         annotator = self.bookkeeper.annotator
-        result = annotator.recursivecall(graph, whence, inputcells, v_result)
+        if whence is not None:
+            annotator.record_call(graph, whence)
+        if v_result is not None:
+            # annotator.notify[graph.returnblock] is a set of variables to update
+            # whenever the return block of this graph has been analysed.
+            returnvars = annotator.notify.setdefault(graph.returnblock, set())
+            returnvars.add(v_result)
+
+        # generalize the function's input arguments
+        annotator.addpendinggraph(graph, inputcells)
+
+        result = graph.getreturnvar().annotation
+        if result is None:
+            result = s_ImpossibleValue
         signature = getattr(self.pyobj, '_signature_', None)
         if signature:
             sigresult = enforce_signature_return(self, signature[1], result)

From pypy.commits at gmail.com  Fri Nov 25 08:31:09 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:09 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Reimplement enforcing of signature
 return values using add_notification()
Message-ID: <58383d1d.8d071c0a.3f205.b3f0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88667:0d45833ec144
Date: 2016-11-25 13:29 +0000
http://bitbucket.org/pypy/pypy/changeset/0d45833ec144/

Log:	Reimplement enforcing of signature return values using
	add_notification()

diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -1,7 +1,8 @@
 from __future__ import absolute_import
 import types
 from rpython.annotator.signature import (
-    enforce_signature_args, enforce_signature_return, finish_type)
+    enforce_signature_args, enforce_signature_return, finish_type,
+    SignatureReturn)
 from rpython.flowspace.model import FunctionGraph
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
@@ -245,6 +246,14 @@
                 postfix = valid_identifier(nameof(key))
                 alt_name = "%s__%s" % (self.name, postfix)
             graph = self.buildgraph(alt_name, builder)
+            signature = getattr(self.pyobj, '_signature_', None)
+            if signature:
+                s_sig = finish_type(signature[1], self.bookkeeper, self.pyobj)
+                if s_sig:
+                    self.bookkeeper.annotator.add_notification(
+                        graph.getreturnvar(), SignatureReturn(self, s_sig))
+                    self.bookkeeper.annotator.addpendingblock(
+                        graph, graph.returnblock, [s_sig])
             self._cache[key] = graph
             return graph
 
@@ -301,13 +310,6 @@
         result = graph.getreturnvar().annotation
         if result is None:
             result = s_ImpossibleValue
-        signature = getattr(self.pyobj, '_signature_', None)
-        if signature:
-            sigresult = enforce_signature_return(self, signature[1], result)
-            if sigresult is not None:
-                annotator.addpendingblock(
-                    graph, graph.returnblock, [sigresult])
-                result = sigresult
         return result
 
     def normalize_args(self, inputs_s):
@@ -345,12 +347,6 @@
         # recreate the args object because inputcells may have been changed
         new_args = args.unmatch_signature(self.signature, inputcells)
         inputcells = self.parse_arguments(new_args, graph)
-        signature = getattr(self.pyobj, '_signature_', None)
-        if signature:
-            s_result = finish_type(signature[1], self.bookkeeper, self.pyobj)
-            if s_result is not None:
-                self.bookkeeper.annotator.addpendingblock(
-                    graph, graph.returnblock, [s_result])
         return graph, inputcells
 
     def bind_under(self, classdef, name):
diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py
--- a/rpython/rlib/test/test_signature.py
+++ b/rpython/rlib/test/test_signature.py
@@ -83,15 +83,6 @@
     def str_to_int(s):
         return s
 
-    @signature(returns=types.str())
-    def str_not_None():
-        return None
-    @check_annotator_fails
-    def caller_of_str_not_None():
-        return str_not_None()
-
- at py.test.mark.xfail
-def test_return_errors_xfail():
     @check_annotator_fails
     @signature(returns=types.str())
     def str_not_None():
@@ -268,11 +259,10 @@
     sig = getsig(g)
     assert sig == [model.SomeFloat(), model.SomeFloat()]
 
+    @check_annotator_fails
     @signature(types.str(), returns=types.int())
     def cannot_add_string(x):
         return f(x, 2)
-    exc = py.test.raises(model.AnnotatorError, annotate_at, cannot_add_string).value
-    assert 'Blocked block' in str(exc)
 
 def test_return_any():
     @signature(types.int(), returns=types.any())

From pypy.commits at gmail.com  Fri Nov 25 08:31:06 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:06 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: create check_signature_return()
Message-ID: <58383d1a.4673c20a.e67f3.f15f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88665:f8f7fa84fee7
Date: 2016-11-25 11:08 +0000
http://bitbucket.org/pypy/pypy/changeset/f8f7fa84fee7/

Log:	create check_signature_return()

diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py
--- a/rpython/annotator/signature.py
+++ b/rpython/annotator/signature.py
@@ -166,8 +166,20 @@
 
 def enforce_signature_return(funcdesc, sigtype, inferredtype):
     s_sigret = finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj)
-    if s_sigret is not None and not s_sigret.contains(inferredtype):
+    if s_sigret is not None:
+        check_signature_return(funcdesc, s_sigret, inferredtype)
+    return s_sigret
+
+def check_signature_return(funcdesc, s_sigret, s_inferred):
+    if not s_sigret.contains(s_inferred):
         raise SignatureError("%r return value:\n"
                         "expected %s,\n"
-                        "     got %s" % (funcdesc, s_sigret, inferredtype))
-    return s_sigret
+                        "     got %s" % (funcdesc, s_sigret, s_inferred))
+
+class SignatureReturn(object):
+    def __init__(self, funcdesc, s_sig):
+        self.funcdesc = funcdesc
+        self.s_sig = s_sig
+
+    def require_update(self, annotator, s_new):
+        check_signature_return(self.funcdesc, self.s_sig, s_new)
diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py
--- a/rpython/rlib/test/test_signature.py
+++ b/rpython/rlib/test/test_signature.py
@@ -26,8 +26,9 @@
     return sigof(a, f)
 
 def check_annotator_fails(caller):
-    exc = py.test.raises(model.AnnotatorError, annotate_at, caller).value
-    assert caller.func_name in str(exc)
+    with py.test.raises(model.AnnotatorError) as excinfo:
+        annotate_at(caller)
+    assert caller.func_name in str(excinfo.value)
 
 
 def test_bookkeeping():

From pypy.commits at gmail.com  Fri Nov 25 08:31:08 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 05:31:08 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: Notify the variable update just after
 the variable is actually updated
Message-ID: <58383d1c.54b31c0a.415cd.b337@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88666:00dc67917db7
Date: 2016-11-25 11:20 +0000
http://bitbucket.org/pypy/pypy/changeset/00dc67917db7/

Log:	Notify the variable update just after the variable is actually
	updated

diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -298,9 +298,17 @@
         successors = self.notify.setdefault(v_src, set())
         successors.add(v_target)
 
-    def update_var(self, v):
-        position_key = self.var_def[v]
-        self.reflowfromposition(position_key)
+    def notify_updated(self, v):
+        if v in self.notify:
+            for v2 in self.notify[v]:
+                self.update_var(v2, v.annotation)
+
+    def update_var(self, v, s_new):
+        if isinstance(v, Variable):
+            position_key = self.var_def[v]
+            self.reflowfromposition(position_key)
+        else:
+            v.require_update(self, s_new)
 
     def reflowfromposition(self, position_key):
         graph, block, index = position_key
@@ -386,11 +394,12 @@
         self.annotated[block] = False  # must re-flow
         self.blocked_blocks[block] = (graph, None)
 
-    def bindinputargs(self, graph, block, inputcells):
+    def bindinputargs(self, graph, block, inputs_s):
         # Create the initial bindings for the input args of a block.
-        assert len(block.inputargs) == len(inputcells)
-        for a, cell in zip(block.inputargs, inputcells):
-            self.setbinding(a, cell)
+        assert len(block.inputargs) == len(inputs_s)
+        for v, s_input in zip(block.inputargs, inputs_s):
+            self.setbinding(v, s_input)
+            self.notify_updated(v)
         self.annotated[block] = False  # must flowin.
         self.blocked_blocks[block] = (graph, None)
 
@@ -453,10 +462,6 @@
         return repr(graph) + blk + opid
 
     def flowin(self, graph, block):
-        for v in block.inputargs:
-            if v in self.notify:
-                for v2 in self.notify[v]:
-                    self.update_var(v2)
         try:
             i = 0
             while i < len(block.operations):

From pypy.commits at gmail.com  Fri Nov 25 08:55:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 05:55:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Trying to avoid some of the dummy
 setfield_gc
Message-ID: <583842e0.43e61c0a.513b1.c280@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88668:11c7fca5679d
Date: 2016-11-25 12:04 +0100
http://bitbucket.org/pypy/pypy/changeset/11c7fca5679d/

Log:	Trying to avoid some of the dummy setfield_gc

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -81,7 +81,8 @@
         try:
             identifier = jit.conditional_call_elidable(
                                 self._utf8, g_encode_utf8, self._value)
-            self._utf8 = identifier
+            if not jit.isconstant(self):
+                self._utf8 = identifier
         except UnicodeEncodeError:
             # bah, this is just to get an official app-level
             # UnicodeEncodeError

From pypy.commits at gmail.com  Fri Nov 25 08:55:46 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 05:55:46 -0800 (PST)
Subject: [pypy-commit] pypy default: Tentatively detect cases of recreation
 of a PyPy object from the
Message-ID: <583842e2.46831c0a.c8125.c62a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88669:4540fd0f69d1
Date: 2016-11-25 14:54 +0100
http://bitbucket.org/pypy/pypy/changeset/4540fd0f69d1/

Log:	Tentatively detect cases of recreation of a PyPy object from the
	PyObject during tp_dealloc.

diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,7 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import fatalerror
 
 
 #________________________________________________________
@@ -192,6 +193,8 @@
     rawrefcount.create_link_pypy(w_obj, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 def from_ref(space, ref):
     """
     Finds the interpreter object corresponding to the given reference.  If the
@@ -202,7 +205,23 @@
         return None
     w_obj = rawrefcount.to_obj(W_Root, ref)
     if w_obj is not None:
-        return w_obj
+        if w_obj is not w_marker_deallocating:
+            return w_obj
+        fatalerror(
+            "*** Invalid usage of a dying CPython object ***\n"
+            "\n"
+            "cpyext, the emulation layer, detected that while it is calling\n"
+            "an object's tp_dealloc, the C code calls back a function that\n"
+            "tries to recreate the PyPy version of the object.  Usually it\n"
+            "means that tp_dealloc calls some general PyXxx() API.  It is\n"
+            "a dangerous and potentially buggy thing to do: even in CPython\n"
+            "the PyXxx() function could, in theory, cause a reference to the\n"
+            "object to be taken and stored somewhere, for an amount of time\n"
+            "exceeding tp_dealloc itself.  Afterwards, the object will be\n"
+            "freed, making that reference point to garbage.\n"
+            ">>> PyPy could contain some workaround to still work if\n"
+            "you are lucky, but it is not done so far; better fix the bug in\n"
+            "the CPython extension.")
 
     # This reference is not yet a real interpreter object.
     # Realize it.
@@ -233,7 +252,8 @@
 INTERPLEVEL_API['as_pyobj'] = as_pyobj
 
 def pyobj_has_w_obj(pyobj):
-    return rawrefcount.to_obj(W_Root, pyobj) is not None
+    w_obj = rawrefcount.to_obj(W_Root, pyobj)
+    return w_obj is not None and w_obj is not w_marker_deallocating
 INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj)
 
 
@@ -335,6 +355,7 @@
     pto = obj.c_ob_type
     #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \
     #      "'s type which is", rffi.charp2str(pto.c_tp_name)
+    rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2936,6 +2936,12 @@
         self._pyobj(pyobject).ob_pypy_link = objint
         # there is no rrc_o_dict
 
+    def rawrefcount_mark_deallocating(self, gcobj, pyobject):
+        ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        obj = llmemory.cast_ptr_to_adr(gcobj)   # should be a prebuilt obj
+        objint = llmemory.cast_adr_to_int(obj, "symbolic")
+        self._pyobj(pyobject).ob_pypy_link = objint
+
     def rawrefcount_from_obj(self, gcobj):
         obj = llmemory.cast_ptr_to_adr(gcobj)
         if self.is_in_nursery(obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -476,6 +476,10 @@
                 GCClass.rawrefcount_create_link_pyobj,
                 [s_gc, s_gcref, SomeAddress()],
                 annmodel.s_None)
+            self.rawrefcount_mark_deallocating = getfn(
+                GCClass.rawrefcount_mark_deallocating,
+                [s_gc, s_gcref, SomeAddress()],
+                annmodel.s_None)
             self.rawrefcount_from_obj_ptr = getfn(
                 GCClass.rawrefcount_from_obj, [s_gc, s_gcref], SomeAddress(),
                 inline = True)
@@ -1281,6 +1285,14 @@
                   [self.rawrefcount_create_link_pyobj_ptr, self.c_const_gc,
                    v_gcobj, v_pyobject])
 
+    def gct_gc_rawrefcount_mark_deallocating(self, hop):
+        [v_gcobj, v_pyobject] = hop.spaceop.args
+        assert v_gcobj.concretetype == llmemory.GCREF
+        assert v_pyobject.concretetype == llmemory.Address
+        hop.genop("direct_call",
+                  [self.rawrefcount_mark_deallocating, self.c_const_gc,
+                   v_gcobj, v_pyobject])
+
     def gct_gc_rawrefcount_from_obj(self, hop):
         [v_gcobj] = hop.spaceop.args
         assert v_gcobj.concretetype == llmemory.GCREF
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -36,6 +36,7 @@
     _pypy2ob = {}
     _pypy2ob_rev = {}
     _d_list = []
+    _d_marker = None
     _dealloc_trigger_callback = dealloc_trigger_callback
 
 @not_rpython
@@ -62,6 +63,14 @@
     _o_list.append(ob)
 
 @not_rpython
+def mark_deallocating(marker, ob):
+    """mark the PyObject as deallocating, by storing 'marker'
+    inside its ob_pypy_link field"""
+    assert ob._obj not in _pypy2ob_rev
+    assert not ob.c_ob_pypy_link
+    ob.c_ob_pypy_link = _build_pypy_link(marker)
+
+ at not_rpython
 def from_obj(OB_PTR_TYPE, p):
     ob = _pypy2ob.get(p)
     if ob is None:
@@ -221,7 +230,7 @@
 
 
 class Entry(ExtRegistryEntry):
-    _about_ = (create_link_pypy, create_link_pyobj)
+    _about_ = (create_link_pypy, create_link_pyobj, mark_deallocating)
 
     def compute_result_annotation(self, s_p, s_ob):
         pass
@@ -231,6 +240,8 @@
             name = 'gc_rawrefcount_create_link_pypy'
         elif self.instance is create_link_pyobj:
             name = 'gc_rawrefcount_create_link_pyobj'
+        elif self.instance is mark_deallocating:
+            name = 'gc_rawrefcount_mark_deallocating'
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -212,6 +212,15 @@
         assert rawrefcount.to_obj(W_Root, ob) == p
         lltype.free(ob, flavor='raw')
 
+    def test_mark_deallocating(self):
+        ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+        w_marker = W_Root(42)
+        rawrefcount.mark_deallocating(w_marker, ob)
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        rawrefcount._collect()
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        lltype.free(ob, flavor='raw')
+
 
 class TestTranslated(StandaloneTests):
 
@@ -222,6 +231,7 @@
         state.seen = []
         def dealloc_trigger():
             state.seen.append(1)
+        w_marker = W_Root(-1)
 
         def make_p():
             p = W_Root(42)
@@ -257,6 +267,13 @@
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
+            if rawrefcount.to_obj(W_Root, ob) is not None:
+                print "to_obj(dead) is not None?"
+                return 1
+            rawrefcount.mark_deallocating(w_marker, ob)
+            if rawrefcount.to_obj(W_Root, ob) is not w_marker:
+                print "to_obj(marked-dead) is not w_marker"
+                return 1
             print "OK!"
             lltype.free(ob, flavor='raw')
             return 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -490,6 +490,7 @@
     'gc_rawrefcount_init':              LLOp(),
     'gc_rawrefcount_create_link_pypy':  LLOp(),
     'gc_rawrefcount_create_link_pyobj': LLOp(),
+    'gc_rawrefcount_mark_deallocating': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
 

From pypy.commits at gmail.com  Fri Nov 25 09:19:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 06:19:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Issue #2355: disable ctypes.pythonapi.
 Failing tests will be fixed when
Message-ID: <5838485f.212dc20a.e6f9c.fcb5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88670:46bb03e8b5d3
Date: 2016-11-25 15:18 +0100
http://bitbucket.org/pypy/pypy/changeset/46bb03e8b5d3/

Log:	Issue #2355: disable ctypes.pythonapi. Failing tests will be fixed
	when we see them.

diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -370,12 +370,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -428,15 +429,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)

From pypy.commits at gmail.com  Fri Nov 25 10:07:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Fri, 25 Nov 2016 07:07:29 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2296: patch for Windows
Message-ID: <583853b1.52301c0a.d82bc.e213@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88671:d6063299a5ce
Date: 2016-11-25 16:06 +0100
http://bitbucket.org/pypy/pypy/changeset/d6063299a5ce/

Log:	Issue #2296: patch for Windows

diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -126,13 +126,14 @@
 // how we'd get the PC (using StackWalk64?)
 //    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
 
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
+// #include "base/logging.h"   // for RAW_LOG
+// #ifndef HAVE_CYGWIN_SIGNAL_H
+// typedef int ucontext_t;
+// #endif
 
 intptr_t GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  fprintf(stderr, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
 

From pypy.commits at gmail.com  Fri Nov 25 13:49:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 10:49:50 -0800 (PST)
Subject: [pypy-commit] pypy desc-specialize: Remove unused imports
Message-ID: <583887ce.4673c20a.e67f3.7925@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: desc-specialize
Changeset: r88672:d997190d59ae
Date: 2016-11-25 18:48 +0000
http://bitbucket.org/pypy/pypy/changeset/d997190d59ae/

Log:	Remove unused imports

diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -7,7 +7,6 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -4,7 +4,6 @@
 sys.path.insert(0, str(ROOT))
 
 import time
-from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
 from rpython.rlib.objectmodel import specialize, dont_inline
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -36,8 +36,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -985,7 +983,7 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -8,7 +8,6 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -8,7 +8,6 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 

From pypy.commits at gmail.com  Fri Nov 25 13:51:03 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 10:51:03 -0800 (PST)
Subject: [pypy-commit] pypy desc-specialize: Close branch desc-specialize
Message-ID: <58388817.aaa3c20a.7ac19.7c8c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: desc-specialize
Changeset: r88673:415819e4eee3
Date: 2016-11-25 18:50 +0000
http://bitbucket.org/pypy/pypy/changeset/415819e4eee3/

Log:	Close branch desc-specialize


From pypy.commits at gmail.com  Fri Nov 25 13:53:50 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 10:53:50 -0800 (PST)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <583888be.0777c20a.5c9e4.7583@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88675:18393cbf9171
Date: 2016-11-25 18:53 +0000
http://bitbucket.org/pypy/pypy/changeset/18393cbf9171/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,7 @@
 
 Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
 call this function" returning a result.
+
+.. branch: desc-specialize
+
+Refactor FunctionDesc.specialize() and related code (RPython annotator).

From pypy.commits at gmail.com  Fri Nov 25 13:56:28 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 10:56:28 -0800 (PST)
Subject: [pypy-commit] pypy reflowing: hg merge default
Message-ID: <5838895c.542e1c0a.d6cc.416d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: reflowing
Changeset: r88676:a7f16e7b539a
Date: 2016-11-25 18:55 +0000
http://bitbucket.org/pypy/pypy/changeset/a7f16e7b539a/

Log:	hg merge default

diff too long, truncating to 2000 out of 2919 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,18 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -118,30 +124,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,3 +20,12 @@
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt
+
+.. branch: conditional_call_value_4
+
+Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
+call this function" returning a result.
+
+.. branch: desc-specialize
+
+Refactor FunctionDesc.specialize() and related code (RPython annotator).
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -120,8 +120,8 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
-    def lookup_in_type_where(self, cls, name):
-        return 'hopefully not needed', getattr(cls, name)
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
 
     def get_and_call_function(self, w_descr, w_obj, *args):
         return w_descr.__get__(w_obj)(*args)
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -7,7 +7,6 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -4,7 +4,6 @@
 sys.path.insert(0, str(ROOT))
 
 import time
-from rpython.rlib.objectmodel import specialize
 from pypy.interpreter.error import OperationError
 from pypy.module._pypyjson.interp_decoder import loads
 from rpython.rlib.objectmodel import specialize, dont_inline
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -36,8 +36,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -985,7 +983,7 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -62,12 +62,14 @@
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
+    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.ensure_object_strategy()  # make sure we can return a borrowed obj
-    # XXX ^^^ how does this interact with CPyListStrategy?
+    cpy_strategy = space.fromcache(CPyListStrategy)
+    if w_list.strategy is not cpy_strategy:
+        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
     w_res = w_list.getitem(index)
     return w_res     # borrowed ref
 
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -124,11 +124,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.bytesobject import PyString_AsString
+    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,7 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import fatalerror
 
 
 #________________________________________________________
@@ -192,6 +193,8 @@
     rawrefcount.create_link_pypy(w_obj, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 def from_ref(space, ref):
     """
     Finds the interpreter object corresponding to the given reference.  If the
@@ -202,7 +205,23 @@
         return None
     w_obj = rawrefcount.to_obj(W_Root, ref)
     if w_obj is not None:
-        return w_obj
+        if w_obj is not w_marker_deallocating:
+            return w_obj
+        fatalerror(
+            "*** Invalid usage of a dying CPython object ***\n"
+            "\n"
+            "cpyext, the emulation layer, detected that while it is calling\n"
+            "an object's tp_dealloc, the C code calls back a function that\n"
+            "tries to recreate the PyPy version of the object.  Usually it\n"
+            "means that tp_dealloc calls some general PyXxx() API.  It is\n"
+            "a dangerous and potentially buggy thing to do: even in CPython\n"
+            "the PyXxx() function could, in theory, cause a reference to the\n"
+            "object to be taken and stored somewhere, for an amount of time\n"
+            "exceeding tp_dealloc itself.  Afterwards, the object will be\n"
+            "freed, making that reference point to garbage.\n"
+            ">>> PyPy could contain some workaround to still work if\n"
+            "you are lucky, but it is not done so far; better fix the bug in\n"
+            "the CPython extension.")
 
     # This reference is not yet a real interpreter object.
     # Realize it.
@@ -233,7 +252,8 @@
 INTERPLEVEL_API['as_pyobj'] = as_pyobj
 
 def pyobj_has_w_obj(pyobj):
-    return rawrefcount.to_obj(W_Root, pyobj) is not None
+    w_obj = rawrefcount.to_obj(W_Root, pyobj)
+    return w_obj is not None and w_obj is not w_marker_deallocating
 INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj)
 
 
@@ -335,6 +355,7 @@
     pto = obj.c_ob_type
     #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \
     #      "'s type which is", rffi.charp2str(pto.c_tp_name)
+    rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -8,7 +8,6 @@
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -12,72 +12,52 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_str,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_str, '__getitem__')
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_unicode,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_unicode, '__getitem__')
 
 @specialize.memo()
 def dict_getitem(space):
     "Utility that returns the app-level descriptor dict.__getitem__."
-    w_src, w_getitem = space.lookup_in_type_where(space.w_dict,
-                                                  '__getitem__')
-    return w_getitem
+    return space.lookup_in_type(space.w_dict, '__getitem__')
 
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -339,6 +339,9 @@
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -8,7 +8,6 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -339,6 +339,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -659,4 +663,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -326,7 +327,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -789,6 +789,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -811,6 +815,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -663,6 +664,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return compute_hash(hlstr(s))
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return compute_hash(hlunicode(u))
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,9 +9,9 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
@@ -347,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -595,33 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
-        descr = op.getdescr()
-        if descr is None:
-            return
-        assert isinstance(descr, CallDescr)
-        effectinfo = descr.get_extra_info()
-        if effectinfo is None or effectinfo.call_shortcut is None:
-            return
-        if op.type == 'r':
-            cond_call_opnum = rop.COND_CALL_VALUE_R
-        elif op.type == 'i':
-            cond_call_opnum = rop.COND_CALL_VALUE_I
-        else:
-            return
-        cs = effectinfo.call_shortcut
-        ptr_box = op.getarg(1 + cs.argnum)
-        if cs.fielddescr is not None:
-            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                           raw=(ptr_box.type == 'i'))
-        else:
-            value_box = ptr_box
-        self.replace_op_with(op, ResOperation(cond_call_opnum,
-                                              [value_box] + op.getarglist(),
-                                              descr=descr))
-
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,8 +1,7 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr, get_call_descr
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -81,21 +80,6 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
-        call_shortcut = CallShortcut(0, tzdescr)
-        effectinfo = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut)
-        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
-            [lltype.Ptr(T)], lltype.Signed,
-            effectinfo)
-        call_shortcut_2 = CallShortcut(0, None)
-        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut_2)
-        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
-            [lltype.Signed], lltype.Signed,
-            effectinfo_2)
-        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -216,7 +200,6 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
-    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1239,6 +1222,10 @@
                        'i3 = gc_load_i(p0,'
                                  '%(strlendescr.offset)s,'
                                  '%(strlendescr.field_size)s)'],
+        [True,  (1,),  'i3 = strhash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(strhashdescr.offset)s,'
+                                 '-%(strhashdescr.field_size)s)'],
         #[False, (1,),  'i3 = unicodelen(p0)' '->'
         #               'i3 = gc_load_i(p0,'
         #                       '%(unicodelendescr.offset)s,'
@@ -1247,7 +1234,10 @@
                        'i3 = gc_load_i(p0,'
                                '%(unicodelendescr.offset)s,'
                                '%(unicodelendescr.field_size)s)'],
-
+        [True,  (1,),  'i3 = unicodehash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(unicodehashdescr.offset)s,'
+                                 '-%(unicodehashdescr.field_size)s)'],
         ## getitem str/unicode
         [True,  (2,4), 'i3 = unicodegetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,'
@@ -1446,26 +1436,3 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
-
-    def test_handle_call_shortcut(self):
-        self.check_rewrite("""
-            [p0]
-            i1 = call_i(123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """, """
-            [p0]
-            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
-            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """)
-
-    def test_handle_call_shortcut_2(self):
-        self.check_rewrite("""
-            [i0]
-            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """, """
-            [i0]
-            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,7 +16,6 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
-    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -656,10 +656,12 @@
     _COND_CALL_SAVE_REGS = [r.r3, r.r4, r.r5, r.r6, r.r12]
 
     def emit_cond_call(self, op, arglocs, regalloc):
+        resloc = arglocs[0]
+        arglocs = arglocs[1:]
+
         fcond = self.guard_success_cc
         self.guard_success_cc = c.cond_none
         assert fcond != c.cond_none
-        fcond = c.negate(fcond)
 
         jmp_adr = self.mc.get_relative_pos()
         self.mc.trap()        # patched later to a 'bc'
@@ -691,7 +693,10 @@
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.bl_abs(cond_call_adr)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  If we have a result, move
+        # it from r2 to its expected location.
+        if resloc is not None:
+            self.mc.mr(resloc.value, r.SCRATCH2.value)
         relative_target = self.mc.currpos() - jmp_adr
         pmc = OverwritingBuilder(self.mc, jmp_adr, 1)
         BI, BO = c.encoding[fcond]
@@ -701,6 +706,9 @@
         # guard_no_exception too
         self.previous_cond_call_jcond = jmp_adr, BI, BO
 
+    emit_cond_call_value_i = emit_cond_call
+    emit_cond_call_value_r = emit_cond_call
+
 
 class FieldOpAssembler(object):
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -315,6 +315,7 @@
         #   * r2 is the gcmap
         #   * the old value of these regs must already be stored in the jitframe
         #   * on exit, all registers are restored from the jitframe
+        #   * the result of the call, if any, is moved to r2
 
         mc = PPCBuilder()
         self.mc = mc
@@ -347,7 +348,11 @@
         # Finish
         self._reload_frame_if_necessary(mc)
 
+        # Move the result, if any, to r2
+        mc.mr(r.SCRATCH2.value, r.r3.value)
+
         mc.mtlr(r.RCS1.value)     # restore LR
+
         self._pop_core_regs_from_jitframe(mc, saved_regs)
         if supports_floats:
             self._pop_fp_regs_from_jitframe(mc)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1038,14 +1038,34 @@
 
     def prepare_cond_call(self, op):
         self.load_condition_into_cc(op.getarg(0))
-        locs = []
+        self.assembler.guard_success_cc = c.negate(
+            self.assembler.guard_success_cc)
+        # ^^^ if arg0==0, we jump over the next block of code (the call)
+        locs = [None]
         # support between 0 and 4 integer arguments
         assert 2 <= op.numargs() <= 2 + 4
         for i in range(1, op.numargs()):
             loc = self.loc(op.getarg(i))
             assert loc.type != FLOAT
             locs.append(loc)
-        return locs
+        return locs     # [None, function, args...]
+
+    def prepare_cond_call_value_i(self, op):
+        x = self.ensure_reg(op.getarg(0))
+        self.load_condition_into_cc(op.getarg(0))
+        self.rm.force_allocate_reg(op, selected_reg=x)   # spilled if survives
+        # ^^^ if arg0!=0, we jump over the next block of code (the call)
+        locs = [x]
+        # support between 0 and 4 integer arguments
+        assert 2 <= op.numargs() <= 2 + 4
+        for i in range(1, op.numargs()):
+            loc = self.loc(op.getarg(i))
+            assert loc.type != FLOAT
+            locs.append(loc)
+        return locs     # [res, function, args...]
+
+    prepare_cond_call_value_r = prepare_cond_call_value_i
+
 
 def notimplemented(self, op):
     msg = '[PPC/regalloc] %s not implemented\n' % op.getopname()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,9 +2448,6 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
-        if not self.cpu.supports_cond_call_value:
-            py.test.skip("missing supports_cond_call_value")
-
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -711,11 +711,6 @@
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
 
-    def filter(self, builder):
-        if not builder.cpu.supports_cond_call_value and \
-           self.opnum == rop.COND_CALL_VALUE_I:
-            raise test_random.CannotProduceOperation
-
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -20,11 +20,6 @@
     r = Random()
     r.jumpahead(piece*99999999)
     OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,7 +16,6 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
-    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.CallTest):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,10 +7,9 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection, CallShortcut)
+    EffectInfo, CallInfoCollection)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
-from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -215,7 +214,6 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
-        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -230,12 +228,6 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
-            if hasattr(funcobj, 'graph'):
-                call_shortcut = self.find_call_shortcut(funcobj.graph)
-            if getattr(func, "_call_shortcut_", False):
-                assert call_shortcut is not None, (
-                    "%r: marked as @jit.call_shortcut but shortcut not found"
-                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -250,8 +242,6 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
-                if hasattr(graph.func, '_call_shortcut_'):
-                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -308,7 +298,6 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
-            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -379,76 +368,3 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
-
-    def find_call_shortcut(self, graph):
-        """Identifies graphs that start like this:
-
-           def graph(x, y, z):         def graph(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r: return r
-        """
-        block = graph.startblock
-        operations = block.operations
-        c_fieldname = None
-        if not operations:
-            v_inst = v_result = block.exitswitch
-        else:
-            op = operations[0]
-            if len(op.args) == 0:
-                return
-            if op.opname != 'getfield':  # check for this form:
-                v_inst = op.args[0]      #     if y is not None;
-                v_result = v_inst        #          return y
-            else:
-                operations = operations[1:]
-                [v_inst, c_fieldname] = op.args
-                v_result = op.result
-        if not isinstance(v_inst, Variable):
-            return
-        if v_result.concretetype != graph.getreturnvar().concretetype:
-            return
-        if v_result.concretetype == lltype.Void:
-            return
-        argnum = i = 0
-        while block.inputargs[i] is not v_inst:
-            if block.inputargs[i].concretetype != lltype.Void:
-                argnum += 1
-            i += 1
-        PSTRUCT = v_inst.concretetype
-        v_check = v_result
-        fastcase = True
-        for op in operations:
-            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
-                    and v_check is op.args[0]):
-                v_check = op.result
-            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
-                v_check = op.result
-                fastcase = not fastcase
-            elif (op.opname in ('int_eq', 'int_ne')
-                    and v_check is op.args[0]
-                    and isinstance(op.args[1], Constant)
-                    and op.args[1].value == 0):
-                v_check = op.result
-                if op.opname == 'int_eq':
-                    fastcase = not fastcase
-            else:
-                return
-        if v_check.concretetype is not lltype.Bool:
-            return
-        if block.exitswitch is not v_check:
-            return
-
-        links = [link for link in block.exits if link.exitcase == fastcase]
-        if len(links) != 1:
-            return
-        [link] = links
-        if link.args != [v_result]:
-            return
-        if not link.target.is_final_block():
-            return
-
-        if c_fieldname is not None:
-            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
-        else:
-            fielddescr = None
-        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,8 +117,7 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True,
-                call_shortcut=None):
+                can_collect=True):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -136,8 +135,7 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect,
-               call_shortcut)
+               can_collect)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -192,7 +190,6 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
-        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -278,8 +275,7 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True,
-                                 call_shortcut=None):
+                                 can_collect=True):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -368,8 +364,7 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect,
-                      call_shortcut)
+                      can_collect)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -392,24 +387,6 @@
 
 # ____________________________________________________________
 
-
-class CallShortcut(object):
-    def __init__(self, argnum, fielddescr):
-        self.argnum = argnum
-        self.fielddescr = fielddescr
-
-    def __eq__(self, other):
-        return (isinstance(other, CallShortcut) and
-                self.argnum == other.argnum and
-                self.fielddescr == other.fielddescr)
-    def __ne__(self, other):
-        return not (self == other)
-    def __hash__(self):
-        return hash((self.argnum, self.fielddescr))
-
-# ____________________________________________________________
-
-
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -789,12 +789,20 @@
                                                 arrayfielddescr,
                                                 arraydescr)
             return []
+        # check for the string or unicode hash field
+        STRUCT = v_inst.concretetype.TO
+        if STRUCT == rstr.STR:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('strhash', [v_inst], op.result)
+        elif STRUCT == rstr.UNICODE:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('unicodehash', [v_inst], op.result)
         # check for _immutable_fields_ hints
-        immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+        immut = STRUCT._immutable_field(c_fieldname.value)
         need_live = False
         if immut:
             if (self.callcontrol is not None and
-                self.callcontrol.could_be_green_field(v_inst.concretetype.TO,
+                self.callcontrol.could_be_green_field(STRUCT,
                                                       c_fieldname.value)):
                 pure = '_greenfield'
                 need_live = True
@@ -802,10 +810,9 @@
                 pure = '_pure'
         else:
             pure = ''
-        self.check_field_access(v_inst.concretetype.TO)
-        argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
-        descr = self.cpu.fielddescrof(v_inst.concretetype.TO,
-                                      c_fieldname.value)
+        self.check_field_access(STRUCT)
+        argname = getattr(STRUCT, '_gckind', 'gc')
+        descr = self.cpu.fielddescrof(STRUCT, c_fieldname.value)
         kind = getkind(RESULT)[0]
         if argname != 'gc':
             assert argname == 'raw'
@@ -822,7 +829,7 @@
         if immut in (IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY):
             op1.opname += "_pure"
             descr1 = self.cpu.fielddescrof(
-                v_inst.concretetype.TO,
+                STRUCT,
                 quasiimmut.get_mutate_field_name(c_fieldname.value))
             return [SpaceOperation('-live-', [], None),
                    SpaceOperation('record_quasiimmut_field',
@@ -1569,7 +1576,7 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
+    def _rewrite_op_cond_call(self, op, rewritten_opname):
         have_floats = False
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
@@ -1580,13 +1587,18 @@
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
-        op1 = self.rewrite_call(op, 'conditional_call',
+        op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
                                 calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_conditional_call(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call')
+    def rewrite_op_jit_conditional_call_value(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call_value')
+
     def handle_jit_marker__jit_merge_point(self, op, jitdriver):
         assert self.portal_jd is not None, (
             "'jit_merge_point' in non-portal graph!")
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+from rpython.jit.codewriter.effectinfo import EffectInfo
 
 
 class FakePolicy:
@@ -368,121 +368,3 @@
         assert call_op.opname == 'direct_call'
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.check_can_collect() == expected
-
-def test_find_call_shortcut():
-    class FakeCPU:
-        def fielddescrof(self, TYPE, fieldname):
-            if isinstance(TYPE, lltype.GcStruct):
-                if fieldname == 'inst_foobar':
-                    return 'foobardescr'
-                if fieldname == 'inst_fooref':
-                    return 'foorefdescr'
-            if TYPE == RAW and fieldname == 'x':
-                return 'xdescr'
-            assert False, (TYPE, fieldname)
-    cc = CallControl(FakeCPU())
-
-    class B(object):
-        foobar = 0
-        fooref = None
-
-    def f1(a, b, c):
-        if b.foobar:
-            return b.foobar
-        b.foobar = a + c
-        return b.foobar
-
-    def f2(x, y, z, b):
-        r = b.fooref
-        if r is not None:
-            return r
-        r = b.fooref = B()
-        return r
-
-    class Space(object):
-        def _freeze_(self):
-            return True
-    space = Space()
-
-    def f3(space, b):
-        r = b.foobar
-        if not r:
-            r = b.foobar = 123
-        return r
-
-    def f4(raw):
-        r = raw.x
-        if r != 0:
-            return r
-        raw.x = 123
-        return 123
-    RAW = lltype.Struct('RAW', ('x', lltype.Signed))
-
-    def f5(b):
-        r = b.foobar
-        if r == 0:
-            r = b.foobar = 123
-        return r
-
-    def f6(b):
-        if b is not None:
-            return b
-        return B()
-
-    def f7(c, a):
-        if a:
-            return a
-        return 123
-
-    def b_or_none(c):
-        if c > 15:
-            return B()
-        return None
-
-    def f(a, c):
-        b = B()
-        f1(a, b, c)
-        f2(a, c, a, b)
-        f3(space, b)
-        r = lltype.malloc(RAW, flavor='raw')
-        f4(r)
-        f5(b)
-        f6(b_or_none(c))
-        f7(c, a)
-
-    rtyper = support.annotate(f, [10, 20])
-    f1_graph = rtyper.annotator.translator._graphof(f1)
-    assert cc.find_call_shortcut(f1_graph) == CallShortcut(1, "foobardescr")
-    f2_graph = rtyper.annotator.translator._graphof(f2)
-    assert cc.find_call_shortcut(f2_graph) == CallShortcut(3, "foorefdescr")
-    f3_graph = rtyper.annotator.translator._graphof(f3)
-    assert cc.find_call_shortcut(f3_graph) == CallShortcut(0, "foobardescr")
-    f4_graph = rtyper.annotator.translator._graphof(f4)
-    assert cc.find_call_shortcut(f4_graph) == CallShortcut(0, "xdescr")
-    f5_graph = rtyper.annotator.translator._graphof(f5)
-    assert cc.find_call_shortcut(f5_graph) == CallShortcut(0, "foobardescr")
-    f6_graph = rtyper.annotator.translator._graphof(f6)
-    assert cc.find_call_shortcut(f6_graph) == CallShortcut(0, None)
-    f7_graph = rtyper.annotator.translator._graphof(f7)
-    assert cc.find_call_shortcut(f7_graph) == CallShortcut(1, None)
-
-def test_cant_find_call_shortcut():
-    from rpython.jit.backend.llgraph.runner import LLGraphCPU
-
-    @jit.dont_look_inside
-    @jit.call_shortcut
-    def f1(n):
-        return n + 17   # no call shortcut found
-
-    def f(n):
-        return f1(n)
-
-    rtyper = support.annotate(f, [1])
-    jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
-    cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
-    res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
-    call_op = f_graph.startblock.operations[0]
-    assert call_op.opname == 'direct_call'
-    e = py.test.raises(AssertionError, cc.getcalldescr, call_op)
-    assert "shortcut not found" in str(e.value)
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.history import MissingValue
 from rpython.rlib import longlong2float
 from rpython.rlib.debug import ll_assert, make_sure_not_resized
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
@@ -132,20 +132,8 @@
                 elif argtype == 'I' or argtype == 'R' or argtype == 'F':
                     assert argcodes[next_argcode] == argtype
                     next_argcode = next_argcode + 1
-                    length = ord(code[position])
-                    position += 1
-                    value = []
-                    for i in range(length):
-                        index = ord(code[position+i])
-                        if   argtype == 'I': reg = self.registers_i[index]
-                        elif argtype == 'R': reg = self.registers_r[index]
-                        elif argtype == 'F': reg = self.registers_f[index]
-                        if not we_are_translated():
-                            assert not isinstance(reg, MissingValue), (
-                                name, self.jitcode, position)
-                        value.append(reg)
-                    make_sure_not_resized(value)
-                    position += length
+                    value = self._get_list_of_values(code, position, argtype)
+                    position += 1 + len(value)
                 elif argtype == 'self':
                     value = self
                 elif argtype == 'cpu':
@@ -1199,13 +1187,27 @@
     def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr):
         return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr)
 
-    # conditional calls - note that they cannot return stuff
     @arguments("cpu", "i", "i", "I", "R", "d")
     def bhimpl_conditional_call_ir_v(cpu, condition, func, args_i, args_r,
                                      calldescr):
+        # conditional calls - condition is a flag, and they cannot return stuff
         if condition:
             cpu.bh_call_v(func, args_i, args_r, None, calldescr)
 
+    @arguments("cpu", "i", "i", "I", "R", "d", returns="i")
+    def bhimpl_conditional_call_value_ir_i(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if value == 0:
+            value = cpu.bh_call_i(func, args_i, args_r, None, calldescr)
+        return value
+
+    @arguments("cpu", "r", "i", "I", "R", "d", returns="r")
+    def bhimpl_conditional_call_value_ir_r(cpu, value, func, args_i, args_r,
+                                           calldescr):
+        if not value:
+            value = cpu.bh_call_r(func, args_i, args_r, None, calldescr)
+        return value
+
     @arguments("cpu", "j", "R", returns="i")
     def bhimpl_inline_call_r_i(cpu, jitcode, args_r):
         return cpu.bh_call_i(jitcode.get_fnaddr_as_int(),
@@ -1493,6 +1495,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_strhash(cpu, string):
+        return cpu.bh_strhash(string)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1509,6 +1514,9 @@
     @arguments("cpu", "r", "r", "i", "i", "i")
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
+    @arguments("cpu", "r", returns="i")
+    def bhimpl_unicodehash(cpu, unicode):
+        return cpu.bh_unicodehash(unicode)
 
     @arguments("i", "i")
     def bhimpl_rvmprof_code(leaving, unique_id):
@@ -1637,6 +1645,24 @@
             if box is not None:
                 self.setarg_f(i, box.getfloatstorage())
 
+    @specialize.arg(3)
+    def _get_list_of_values(self, code, position, argtype):
+        length = ord(code[position])
+        position += 1
+        value = []
+        for i in range(length):
+            index = ord(code[position+i])
+            if   argtype == 'I': reg = self.registers_i[index]
+            elif argtype == 'R': reg = self.registers_r[index]
+            elif argtype == 'F': reg = self.registers_f[index]
+            else: assert 0
+            if not we_are_translated():
+                assert not isinstance(reg, MissingValue), (
+                    name, self.jitcode, position)
+            value.append(reg)
+        make_sure_not_resized(value)
+        return value
+
 # ____________________________________________________________
 
 def _run_forever(blackholeinterp, current_exc):
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -271,6 +271,7 @@
             return
         if (OpHelpers.is_plain_call(opnum) or
             OpHelpers.is_call_loopinvariant(opnum) or
+            OpHelpers.is_cond_call_value(opnum) or
             opnum == rop.COND_CALL):
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -1,5 +1,5 @@
 
-from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash
 from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\
      rop, OpHelpers
 from rpython.jit.metainterp.history import ConstInt, Const
@@ -73,6 +73,9 @@
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         return None
 
+    def getstrhash(self, op, mode):
+        return None
+
     def copy_fields_to_const(self, constinfo, optheap):
         pass
 
@@ -790,6 +793,20 @@
                 return None
             return ConstInt(len(s))
 
+    def getstrhash(self, op, mode):
+        from rpython.jit.metainterp.optimizeopt import vstring
+
+        if mode is vstring.mode_string:
+            s = self._unpack_str(vstring.mode_string)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+        else:
+            s = self._unpack_str(vstring.mode_unicode)
+            if s is None:
+                return None
+            return ConstInt(compute_hash(s))
+
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -261,9 +261,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
-    def _can_optimize_call_pure(self, op):
+    def _can_optimize_call_pure(self, op, start_index=0):
         arg_consts = []
-        for i in range(op.numargs()):
+        for i in range(start_index, op.numargs()):
             arg = op.getarg(i)
             const = self.optimizer.get_constant_box(arg)
             if const is None:
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -151,13 +151,13 @@
             self._pure_operations[opnum] = recentops = RecentPureOps()
         return recentops
 
-    def optimize_CALL_PURE_I(self, op):
+    def optimize_call_pure(self, op, start_index=0):
         # Step 1: check if all arguments are constant
-        for arg in op.getarglist():
-            self.optimizer.force_box(arg)
+        for i in range(start_index, op.numargs()):
+            self.optimizer.force_box(op.getarg(i))
             # XXX hack to ensure that virtuals that are
             #     constant are presented that way
-        result = self._can_optimize_call_pure(op)
+        result = self._can_optimize_call_pure(op, start_index=start_index)
         if result is not None:
             # this removes a CALL_PURE with all constant arguments.
             self.make_constant(op, result)
@@ -168,32 +168,46 @@
         # CALL_PURE.
         for pos in self.call_pure_positions:
             old_op = self.optimizer._newoperations[pos]
-            if self.optimize_call_pure(op, old_op):
+            if self.optimize_call_pure_old(op, old_op, start_index):
                 return
         if self.extra_call_pure:
             for i, old_op in enumerate(self.extra_call_pure):
-                if self.optimize_call_pure(op, old_op):
+                if self.optimize_call_pure_old(op, old_op, start_index):
                     if isinstance(old_op, PreambleOp):
                         old_op = self.optimizer.force_op_from_preamble(old_op)
                         self.extra_call_pure[i] = old_op
                     return
 
-        # replace CALL_PURE with just CALL
-        opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = self.optimizer.replace_op_with(op, opnum)
+        # replace CALL_PURE with just CALL (but keep COND_CALL_VALUE)
+        if start_index == 0:
+            opnum = OpHelpers.call_for_descr(op.getdescr())
+            newop = self.optimizer.replace_op_with(op, opnum)
+        else:
+            newop = op
         return self.emit_result(CallPureOptimizationResult(self, newop))
 
+    def optimize_CALL_PURE_I(self, op):
+        return self.optimize_call_pure(op)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
     optimize_CALL_PURE_N = optimize_CALL_PURE_I
 
-    def optimize_call_pure(self, op, old_op):
-        if (op.numargs() != old_op.numargs() or
-            op.getdescr() is not old_op.getdescr()):
+    def optimize_COND_CALL_VALUE_I(self, op):
+        return self.optimize_call_pure(op, start_index=1)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
+    def optimize_call_pure_old(self, op, old_op, start_index):
+        if op.getdescr() is not old_op.getdescr():
             return False
-        for i, box in enumerate(old_op.getarglist()):
-            if not self.get_box_replacement(op.getarg(i)).same_box(box):
+        # this will match a call_pure and a cond_call_value with
+        # the same function and arguments
+        j = start_index
+        old_start_index = OpHelpers.is_cond_call_value(old_op.opnum)
+        for i in range(old_start_index, old_op.numargs()):
+            box = old_op.getarg(i)
+            if not self.get_box_replacement(op.getarg(j)).same_box(box):
                 break
+            j += 1
         else:
             # all identical
             # this removes a CALL_PURE that has the same (non-constant)
@@ -250,10 +264,17 @@
             # don't move call_pure_with_exception in the short preamble...
             # issue #2015
 
+            # Also, don't move cond_call_value in the short preamble.
+            # The issue there is that it's usually pointless to try to
+            # because the 'value' argument is typically not a loop
+            # invariant, and would really need to be in order to end up
+            # in the short preamble.  Maybe the code works anyway in the
+            # other rare case, but better safe than sorry and don't try.
             effectinfo = op.getdescr().get_extra_info()
             if not effectinfo.check_can_raise(ignore_memoryerror=True):
                 assert rop.is_call(op.opnum)
-                sb.add_pure_op(op)
+                if not OpHelpers.is_cond_call_value(op.opnum):
+                    sb.add_pure_op(op)
 
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -596,6 +596,19 @@
             op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         return self.emit(op)
 
+    def optimize_COND_CALL_VALUE_I(self, op):
+        # look if we know the nullness of the first argument
+        info = self.getnullness(op.getarg(0))
+        if info == INFO_NONNULL:
+            self.make_equal_to(op, op.getarg(0))
+            self.last_emitted_operation = REMOVED
+            return
+        if info == INFO_NULL:
+            opnum = OpHelpers.call_pure_for_type(op.type)
+            op = self.replace_op_with(op, opnum, args=op.getarglist()[1:])
+        return self.emit(op)
+    optimize_COND_CALL_VALUE_R = optimize_COND_CALL_VALUE_I
+
     def _optimize_nullness(self, op, box, expect_nonnull):
         info = self.getnullness(box)
         if info == INFO_NONNULL:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp import compile, resume
 from rpython.jit.metainterp.history import AbstractDescr, ConstInt, TreeLoop
+from rpython.jit.metainterp.history import ConstPtr
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import build_opt_chain
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
@@ -8674,6 +8675,75 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_cond_call_with_a_constant_i(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(0, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        i2 = call_i(123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_with_a_constant_i2(self):
+        ops = """
+        [p1]
+        i2 = cond_call_value_i(12, 123, p1, descr=plaincalldescr)
+        escape_n(i2)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        escape_n(12)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r1(self):
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        jump(p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_cond_call_r2(self):
+        ops = """
+        [p1]
+        guard_nonnull(p1) []
+        p2 = cond_call_value_r(p1, 123, p1, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        guard_nonnull(p1) []
+        p3 = escape_r(p1)
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_cond_call_r3(self):
+        arg_consts = [ConstInt(i) for i in (123, 4, 5, 6)]
+        call_pure_results = {tuple(arg_consts): ConstPtr(self.myptr)}
+        ops = """
+        [p1]
+        p2 = cond_call_value_r(p1, 123, 4, 5, 6, descr=plain_r_calldescr)
+        p3 = escape_r(p2)
+        jump(p3)
+        """
+        expected = """
+        [p1]
+        p3 = escape_r(ConstPtr(myptr))
+        jump(p3)
+        """
+        self.optimize_loop(ops, expected, call_pure_results=call_pure_results)
+
     def test_hippyvm_unroll_bug(self):
         ops = """
         [p0, i1, i2]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -438,6 +438,10 @@
                     oopspecindex=EffectInfo.OS_INT_PY_MOD)
     int_py_mod_descr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
 
+    FUNC = lltype.FuncType([], llmemory.GCREF)
+    ei = EffectInfo([], [], [], [], [], [], EffectInfo.EF_ELIDABLE_CAN_RAISE)
+    plain_r_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, ei)
+
     namespace = locals()
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -537,6 +537,20 @@
                 return
         return self.emit(op)
 
+    def optimize_STRHASH(self, op):
+        return self._optimize_STRHASH(op, mode_string)
+    def optimize_UNICODEHASH(self, op):
+        return self._optimize_STRHASH(op, mode_unicode)
+
+    def _optimize_STRHASH(self, op, mode):
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo:
+            lgtop = opinfo.getstrhash(op, mode)
+            if lgtop is not None:
+                self.make_equal_to(op, lgtop)
+                return
+        return self.emit(op)
+
     def optimize_COPYSTRCONTENT(self, op):
         return self._optimize_COPYSTRCONTENT(op, mode_string)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1059,8 +1059,21 @@
     @arguments("box", "box", "boxes2", "descr", "orgpc")
     def opimpl_conditional_call_ir_v(self, condbox, funcbox, argboxes,
                                      calldescr, pc):
+        if isinstance(condbox, ConstInt) and condbox.value == 0:
+            return   # so that the heapcache can keep argboxes virtual
         self.do_conditional_call(condbox, funcbox, argboxes, calldescr, pc)
 
+    @arguments("box", "box", "boxes2", "descr", "orgpc")
+    def _opimpl_conditional_call_value(self, valuebox, funcbox, argboxes,
+                                       calldescr, pc):
+        if isinstance(valuebox, Const) and valuebox.nonnull():
+            return valuebox
+        return self.do_conditional_call(valuebox, funcbox, argboxes,
+                                        calldescr, pc, is_value=True)
+
+    opimpl_conditional_call_value_ir_i = _opimpl_conditional_call_value
+    opimpl_conditional_call_value_ir_r = _opimpl_conditional_call_value
+
     @arguments("int", "boxes3", "boxes3", "orgpc")
     def _opimpl_recursive_call(self, jdindex, greenboxes, redboxes, pc):
         targetjitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
@@ -1150,6 +1163,20 @@
         self.execute(rop.UNICODESETITEM, unicodebox, indexbox, newcharbox)
 
     @arguments("box")
+    def opimpl_strhash(self, strbox):
+        if isinstance(strbox, ConstPtr):
+            h = self.metainterp.cpu.bh_strhash(strbox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.STRHASH, strbox)
+
+    @arguments("box")
+    def opimpl_unicodehash(self, unicodebox):
+        if isinstance(unicodebox, ConstPtr):
+            h = self.metainterp.cpu.bh_unicodehash(unicodebox.getref_base())
+            return ConstInt(h)
+        return self.execute(rop.UNICODEHASH, unicodebox)
+
+    @arguments("box")
     def opimpl_newstr(self, lengthbox):
         return self.execute(rop.NEWSTR, lengthbox)
 
@@ -1538,7 +1565,7 @@
                                                             descr=descr)
         if pure and not self.metainterp.last_exc_value and op:
             op = self.metainterp.record_result_of_call_pure(op, argboxes, descr,
-                patch_pos)
+                patch_pos, opnum)
             exc = exc and not isinstance(op, Const)
         if exc:
             if op is not None:
@@ -1712,16 +1739,31 @@
             else:
                 assert False
 
-    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
-        if isinstance(condbox, ConstInt) and condbox.value == 0:
-            return   # so that the heapcache can keep argboxes virtual
+    def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc,
+                            is_value=False):
         allboxes = self._build_allboxes(funcbox, argboxes, descr)
         effectinfo = descr.get_extra_info()
         assert not effectinfo.check_forces_virtual_or_virtualizable()
         exc = effectinfo.check_can_raise()
-        pure = effectinfo.check_is_elidable()
-        return self.execute_varargs(rop.COND_CALL, [condbox] + allboxes, descr,
-                                    exc, pure)
+        allboxes = [condbox] + allboxes
+        # COND_CALL cannot be pure (=elidable): it has no result.
+        # On the other hand, COND_CALL_VALUE is always calling a pure
+        # function.
+        if not is_value:
+            return self.execute_varargs(rop.COND_CALL, allboxes, descr,
+                                        exc, pure=False)
+        else:
+            opnum = OpHelpers.cond_call_value_for_descr(descr)
+            # work around the fact that execute_varargs() wants a
+            # constant for first argument
+            if opnum == rop.COND_CALL_VALUE_I:
+                return self.execute_varargs(rop.COND_CALL_VALUE_I, allboxes,
+                                            descr, exc, pure=True)
+            elif opnum == rop.COND_CALL_VALUE_R:
+                return self.execute_varargs(rop.COND_CALL_VALUE_R, allboxes,
+                                            descr, exc, pure=True)
+            else:
+                raise AssertionError
 
     def _do_jit_force_virtual(self, allboxes, descr, pc):
         assert len(allboxes) == 2
@@ -3061,11 +3103,16 @@
         debug_stop("jit-abort-longest-function")
         return max_jdsd, max_key
 
-    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos):
+    def record_result_of_call_pure(self, op, argboxes, descr, patch_pos, opnum):
         """ Patch a CALL into a CALL_PURE.
         """
         resbox_as_const = executor.constant_from_op(op)
-        for argbox in argboxes:
+        is_cond_value = OpHelpers.is_cond_call_value(opnum)
+        if is_cond_value:
+            normargboxes = argboxes[1:]    # ingore the 'value' arg
+        else:
+            normargboxes = argboxes

From pypy.commits at gmail.com  Fri Nov 25 13:51:21 2016
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 25 Nov 2016 10:51:21 -0800 (PST)
Subject: [pypy-commit] pypy default: Merged in desc-specialize (pull request
 #499)
Message-ID: <58388829.c19d1c0a.22254.494b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r88674:a4e89926bd47
Date: 2016-11-25 18:50 +0000
http://bitbucket.org/pypy/pypy/changeset/a4e89926bd47/

Log:	Merged in desc-specialize (pull request #499)

	Refactor FunctionDesc.specialize() and related code

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -439,6 +439,7 @@
                                           space.wrap(msg))
         return OperationError(exc, w_error)
 
+ at specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
                   w_exception_class=None):
     assert isinstance(e, OSError)
@@ -466,8 +467,8 @@
         w_error = space.call_function(exc, space.wrap(errno),
                                       space.wrap(msg))
     return OperationError(exc, w_error)
-wrap_oserror2._annspecialcase_ = 'specialize:arg(3)'
 
+ at specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
                  w_exception_class=None):
     if filename is not None:
@@ -478,7 +479,6 @@
         return wrap_oserror2(space, e, None,
                              exception_name=exception_name,
                              w_exception_class=w_exception_class)
-wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
@@ -6,7 +7,6 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -124,13 +124,13 @@
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         if can_be_None and w_obj is None:
             return None
         if not isinstance(w_obj, RequiredClass):
             raise TypeError
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def getarg_w(self, code, w_obj):    # for retrieving buffers
         return FakeBuffer(w_obj)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -36,8 +36,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -985,7 +983,7 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
@@ -1614,9 +1612,8 @@
     miniglobals = {'__name__':    __name__, # for module name propagation
                    }
     exec source.compile() in miniglobals
-    call_external_function = miniglobals['cpy_call_external']
+    call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
     call_external_function._dont_inline_ = True
-    call_external_function._annspecialcase_ = 'specialize:ll'
     call_external_function._gctransformer_hint_close_stack_ = True
     # don't inline, as a hack to guarantee that no GC pointer is alive
     # anywhere in call_external_function
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -3,11 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -202,6 +202,7 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):
@@ -215,7 +216,6 @@
                 return w_some_obj()
             self._wrap_not_rpython(x)
         return w_some_obj()
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
@@ -305,10 +305,10 @@
         is_root(w_complex)
         return 1.1, 2.2
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     def decode_index(self, w_index_or_slice, seqlength):
         is_root(w_index_or_slice)
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -9,9 +9,9 @@
 
 def make_checker():
     check = []
+    @specialize.memo()
     def see():
         check.append(True)
-    see._annspecialcase_ = 'specialize:memo'
     return see, check
 
 def test_wrap_interp2app():
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -2,12 +2,12 @@
 import sys
 
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import INT_MAX
 from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -2,6 +2,7 @@
 
 import types
 from collections import defaultdict
+from contextlib import contextmanager
 
 from rpython.tool.ansi_print import AnsiLogger
 from rpython.tool.pairtype import pair
@@ -83,22 +84,17 @@
         annmodel.TLS.check_str_without_nul = (
             self.translator.config.translation.check_str_without_nul)
 
-        flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
+        with self.using_policy(policy):
+            flowgraph, inputs_s = self.get_call_parameters(function, args_s)
 
         if main_entry_point:
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
-    def get_call_parameters(self, function, args_s, policy):
-        desc = self.bookkeeper.getdesc(function)
-        prevpolicy = self.policy
-        self.policy = policy
-        self.bookkeeper.enter(None)
-        try:
+    def get_call_parameters(self, function, args_s):
+        with self.bookkeeper.at_position(None):
+            desc = self.bookkeeper.getdesc(function)
             return desc.get_call_parameters(args_s)
-        finally:
-            self.bookkeeper.leave()
-            self.policy = prevpolicy
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
@@ -107,21 +103,29 @@
             # XXX hack
             annmodel.TLS.check_str_without_nul = (
                 self.translator.config.translation.check_str_without_nul)
-        graph, inputcells = self.get_call_parameters(function, args_s, policy)
-        self.build_graph_types(graph, inputcells, complete_now=False)
-        self.complete_helpers(policy)
+        with self.using_policy(policy):
+            graph, inputcells = self.get_call_parameters(function, args_s)
+            self.build_graph_types(graph, inputcells, complete_now=False)
+            self.complete_helpers()
         return graph
 
-    def complete_helpers(self, policy):
-        saved = self.policy, self.added_blocks
-        self.policy = policy
+    def complete_helpers(self):
+        saved = self.added_blocks
+        self.added_blocks = {}
         try:
-            self.added_blocks = {}
             self.complete()
             # invoke annotation simplifications for the new blocks
             self.simplify(block_subset=self.added_blocks)
         finally:
-            self.policy, self.added_blocks = saved
+            self.added_blocks = saved
+
+    @contextmanager
+    def using_policy(self, policy):
+        """A context manager that temporarily replaces the annotator policy"""
+        old_policy = self.policy
+        self.policy = policy
+        yield
+        self.policy = old_policy
 
     def build_graph_types(self, flowgraph, inputcells, complete_now=True):
         checkgraph(flowgraph)
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -9,6 +9,7 @@
 from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
+from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.model import (
     SomeOrderedDict, SomeString, SomeChar, SomeFloat, unionof, SomeInstance,
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
@@ -21,6 +22,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
+from rpython.annotator.specialize import memo
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
@@ -358,7 +360,7 @@
             return self.descs[obj_key]
         except KeyError:
             if isinstance(pyobj, types.FunctionType):
-                result = description.FunctionDesc(self, pyobj)
+                result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
                     raise Exception("ClassDesc for object not supported")
@@ -403,6 +405,23 @@
             self.descs[obj_key] = result
             return result
 
+    def newfuncdesc(self, pyfunc):
+        name = pyfunc.__name__
+        if hasattr(pyfunc, '_generator_next_method_of_'):
+            from rpython.flowspace.argument import Signature
+            signature = Signature(['entry'])     # haaaaaack
+            defaults = ()
+        else:
+            signature = cpython_code_signature(pyfunc.func_code)
+            defaults = pyfunc.func_defaults
+        # get the specializer based on the tag of the 'pyobj'
+        # (if any), according to the current policy
+        tag = getattr(pyfunc, '_annspecialcase_', None)
+        specializer = self.annotator.policy.get_specializer(tag)
+        if specializer is memo:
+            return description.MemoDesc(self, pyfunc, name, signature, defaults, specializer)
+        return description.FunctionDesc(self, pyfunc, name, signature, defaults, specializer)
+
     def getfrozen(self, pyobj):
         return description.FrozenDesc(self, pyobj)
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -608,7 +608,7 @@
             if mixin:
                 # make a new copy of the FunctionDesc for this class,
                 # but don't specialize further for all subclasses
-                funcdesc = FunctionDesc(self.bookkeeper, value)
+                funcdesc = self.bookkeeper.newfuncdesc(value)
                 self.classdict[name] = funcdesc
                 return
             # NB. if value is, say, AssertionError.__init__, then we
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -3,11 +3,10 @@
 from rpython.annotator.signature import (
     enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import FunctionGraph
-from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, s_ImpossibleValue
+from rpython.annotator.model import AnnotatorError, s_ImpossibleValue, unionof
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -117,7 +116,6 @@
         self.s_value = s_ImpossibleValue    # union of possible values
 
     def update(self, other):
-        from rpython.annotator.model import unionof
         self.descs.update(other.descs)
         self.read_locations.update(other.read_locations)
         self.s_value = unionof(self.s_value, other.s_value)
@@ -192,24 +190,12 @@
 class FunctionDesc(Desc):
     knowntype = types.FunctionType
 
-    def __init__(self, bookkeeper, pyobj=None,
-                 name=None, signature=None, defaults=None,
+    def __init__(self, bookkeeper, pyobj, name, signature, defaults,
                  specializer=None):
         super(FunctionDesc, self).__init__(bookkeeper, pyobj)
-        if name is None:
-            name = pyobj.func_name
-        if signature is None:
-            if hasattr(pyobj, '_generator_next_method_of_'):
-                from rpython.flowspace.argument import Signature
-                signature = Signature(['entry'])     # haaaaaack
-                defaults = ()
-            else:
-                signature = cpython_code_signature(pyobj.func_code)
-        if defaults is None:
-            defaults = pyobj.func_defaults
         self.name = name
         self.signature = signature
-        self.defaults = defaults or ()
+        self.defaults = defaults if defaults is not None else ()
         # 'specializer' is a function with the following signature:
         #      specializer(funcdesc, args_s) => graph
         #                                 or => s_result (overridden/memo cases)
@@ -288,12 +274,43 @@
                 getattr(self.bookkeeper, "position_key", None) is not None):
             _, block, i = self.bookkeeper.position_key
             op = block.operations[i]
-        if self.specializer is None:
-            # get the specializer based on the tag of the 'pyobj'
-            # (if any), according to the current policy
-            tag = getattr(self.pyobj, '_annspecialcase_', None)
-            policy = self.bookkeeper.annotator.policy
-            self.specializer = policy.get_specializer(tag)
+        self.normalize_args(inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
+
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
+        # Some specializations may break the invariant of returning
+        # annotations that are always more general than the previous time.
+        # We restore it here:
+        result = unionof(result, s_previous_result)
+        return result
+
+    def normalize_args(self, inputs_s):
+        """
+        Canonicalize argument annotations into the exact parameter
+        annotations of a specific specialized graph.
+
+        Note: this method has no return value but mutates its argument instead.
+        """
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
@@ -304,39 +321,9 @@
                 from rpython.annotator.signature import Sig
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
-            enforceargs(self, inputcells)  # can modify inputcells in-place
+            enforceargs(self, inputs_s)  # can modify inputs_s in-place
         if signature:
-            enforce_signature_args(self, signature[0], inputcells)  # mutates inputcells
-        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
-            return self.specializer(self, inputcells, op)
-        else:
-            return self.specializer(self, inputcells)
-
-    def pycall(self, whence, args, s_previous_result, op=None):
-        inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
-        # Some specializations may break the invariant of returning
-        # annotations that are always more general than the previous time.
-        # We restore it here:
-        from rpython.annotator.model import unionof
-        result = unionof(result, s_previous_result)
-        return result
+            enforce_signature_args(self, signature[0], inputs_s)  # mutates inputs_s
 
     def get_graph(self, args, op):
         inputs_s = self.parse_arguments(args)
@@ -405,6 +392,16 @@
 
             return s_sigs
 
+class MemoDesc(FunctionDesc):
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        s_result = self.specialize(inputcells, op)
+        if isinstance(s_result, FunctionGraph):
+            s_result = s_result.getreturnvar().annotation
+        s_result = unionof(s_result, s_previous_result)
+        return s_result
+
+
 class MethodDesc(Desc):
     knowntype = types.MethodType
 
diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py
--- a/rpython/annotator/specialize.py
+++ b/rpython/annotator/specialize.py
@@ -3,11 +3,13 @@
 
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import Block, Link, Variable, SpaceOperation
+from rpython.flowspace.model import Block, Link, Variable
 from rpython.flowspace.model import checkgraph
 from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.flowspace.argument import Signature
+from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
+from rpython.annotator.model import unionof
 
 def flatten_star_args(funcdesc, args_s):
     argnames, vararg, kwarg = funcdesc.signature
@@ -127,7 +129,6 @@
     def finish(self):
         if self.do_not_process:
             return
-        from rpython.annotator.model import unionof
         assert self.graph is None, "MemoTable already finished"
         # list of which argument positions can take more than one value
         example_args, example_value = self.table.iteritems().next()
@@ -246,34 +247,36 @@
             args_s.append(unionof(*values_s))
         annotator.addpendinggraph(self.graph, args_s)
 
+def all_values(s):
+    """Return the exhaustive list of possible values matching annotation `s`.
 
-def memo(funcdesc, arglist_s):
-    from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
-    from rpython.annotator.model import unionof
+    Raises `AnnotatorError` if no such (reasonably small) finite list exists.
+    """
+    if s.is_constant():
+        return [s.const]
+    elif isinstance(s, SomePBC):
+        values = []
+        assert not s.can_be_None, "memo call: cannot mix None and PBCs"
+        for desc in s.descriptions:
+            if desc.pyobj is None:
+                raise annmodel.AnnotatorError(
+                    "memo call with a class or PBC that has no "
+                    "corresponding Python object (%r)" % (desc,))
+            values.append(desc.pyobj)
+        return values
+    elif isinstance(s, SomeImpossibleValue):
+        return []
+    elif isinstance(s, SomeBool):
+        return [False, True]
+    else:
+        raise annmodel.AnnotatorError("memo call: argument must be a class "
+                                        "or a frozen PBC, got %r" % (s,))
+
+def memo(funcdesc, args_s):
     # call the function now, and collect possible results
-    argvalues = []
-    for s in arglist_s:
-        if s.is_constant():
-            values = [s.const]
-        elif isinstance(s, SomePBC):
-            values = []
-            assert not s.can_be_None, "memo call: cannot mix None and PBCs"
-            for desc in s.descriptions:
-                if desc.pyobj is None:
-                    raise annmodel.AnnotatorError(
-                        "memo call with a class or PBC that has no "
-                        "corresponding Python object (%r)" % (desc,))
-                values.append(desc.pyobj)
-        elif isinstance(s, SomeImpossibleValue):
-            return s    # we will probably get more possible args later
-        elif isinstance(s, SomeBool):
-            values = [False, True]
-        else:
-            raise annmodel.AnnotatorError("memo call: argument must be a class "
-                                          "or a frozen PBC, got %r" % (s,))
-        argvalues.append(values)
+
     # the list of all possible tuples of arguments to give to the memo function
-    possiblevalues = cartesian_product(argvalues)
+    possiblevalues = cartesian_product([all_values(s_arg) for s_arg in args_s])
 
     # a MemoTable factory -- one MemoTable per family of arguments that can
     # be called together, merged via a UnionFind.
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -138,11 +138,12 @@
         # get the graph of the mix-level helper ll_function and prepare it for
         # being annotated.  Annotation and RTyping should be done in a single shot
         # at the end with finish().
-        graph, args_s = self.rtyper.annotator.get_call_parameters(
-            ll_function, args_s, policy = self.policy)
+        ann = self.rtyper.annotator
+        with ann.using_policy(self.policy):
+            graph, args_s = ann.get_call_parameters(ll_function, args_s)
         for v_arg, s_arg in zip(graph.getargs(), args_s):
-            self.rtyper.annotator.setbinding(v_arg, s_arg)
-        self.rtyper.annotator.setbinding(graph.getreturnvar(), s_result)
+            ann.setbinding(v_arg, s_arg)
+        ann.setbinding(graph.getreturnvar(), s_result)
         #self.rtyper.annotator.annotated[graph.returnblock] = graph
         self.pending.append((ll_function, graph, args_s, s_result))
         return graph
@@ -224,16 +225,17 @@
         bk = ann.bookkeeper
         translator = ann.translator
         original_graph_count = len(translator.graphs)
-        for ll_function, graph, args_s, s_result in self.pending:
-            # mark the return block as already annotated, because the return var
-            # annotation was forced in getgraph() above.  This prevents temporary
-            # less general values reaching the return block from crashing the
-            # annotator (on the assert-that-new-binding-is-not-less-general).
-            ann.annotated[graph.returnblock] = graph
-            s_function = bk.immutablevalue(ll_function)
-            bk.emulate_pbc_call(graph, s_function, args_s)
-            self.newgraphs.add(graph)
-        ann.complete_helpers(self.policy)
+        with ann.using_policy(self.policy):
+            for ll_function, graph, args_s, s_result in self.pending:
+                # mark the return block as already annotated, because the return var
+                # annotation was forced in getgraph() above.  This prevents temporary
+                # less general values reaching the return block from crashing the
+                # annotator (on the assert-that-new-binding-is-not-less-general).
+                ann.annotated[graph.returnblock] = graph
+                s_function = bk.immutablevalue(ll_function)
+                bk.emulate_pbc_call(graph, s_function, args_s)
+                self.newgraphs.add(graph)
+            ann.complete_helpers()
         for ll_function, graph, args_s, s_result in self.pending:
             s_real_result = ann.binding(graph.getreturnvar())
             if s_real_result != s_result:
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -384,7 +384,6 @@
 def _sizeof_none(TYPE):
     assert not TYPE._is_varsize()
     return ItemOffset(TYPE)
-_sizeof_none._annspecialcase_ = 'specialize:memo'
 
 @specialize.memo()
 def _internal_array_field(TYPE):
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -362,9 +362,9 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         funcdesc, = s_pbc.descriptions
-        args = simple_args(args_s)
         with bk.at_position(None):
-            graph = funcdesc.get_graph(args, op)
+            argspec = simple_args(args_s)
+            graph = funcdesc.get_graph(argspec, op)
         llfn = self.rtyper.getcallable(graph)
         return inputconst(typeOf(llfn), llfn)
 
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -846,28 +846,29 @@
         rtyper = self.rtyper
         args_s = []
         newargs_v = []
-        for v in args_v:
-            if v.concretetype is Void:
-                s_value = rtyper.annotation(v)
-                if s_value is None:
-                    s_value = annmodel.s_None
-                if not s_value.is_constant():
-                    raise TyperError("non-constant variable of type Void")
-                if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
-                    raise TyperError("non-PBC Void argument: %r", (s_value,))
-                args_s.append(s_value)
-            else:
-                args_s.append(lltype_to_annotation(v.concretetype))
-            newargs_v.append(v)
+        with rtyper.annotator.using_policy(rtyper.lowlevel_ann_policy):
+            for v in args_v:
+                if v.concretetype is Void:
+                    s_value = rtyper.annotation(v)
+                    if s_value is None:
+                        s_value = annmodel.s_None
+                    if not s_value.is_constant():
+                        raise TyperError("non-constant variable of type Void")
+                    if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
+                        raise TyperError("non-PBC Void argument: %r", (s_value,))
+                    args_s.append(s_value)
+                else:
+                    args_s.append(lltype_to_annotation(v.concretetype))
+                newargs_v.append(v)
 
-        self.rtyper.call_all_setups()  # compute ForwardReferences now
+            self.rtyper.call_all_setups()  # compute ForwardReferences now
 
-        # hack for bound methods
-        if hasattr(ll_function, 'im_func'):
-            bk = rtyper.annotator.bookkeeper
-            args_s.insert(0, bk.immutablevalue(ll_function.im_self))
-            newargs_v.insert(0, inputconst(Void, ll_function.im_self))
-            ll_function = ll_function.im_func
+            # hack for bound methods
+            if hasattr(ll_function, 'im_func'):
+                bk = rtyper.annotator.bookkeeper
+                args_s.insert(0, bk.immutablevalue(ll_function.im_self))
+                newargs_v.insert(0, inputconst(Void, ll_function.im_self))
+                ll_function = ll_function.im_func
 
         graph = annotate_lowlevel_helper(rtyper.annotator, ll_function, args_s,
                                          rtyper.lowlevel_ann_policy)

From pypy.commits at gmail.com  Sat Nov 26 13:53:52 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 26 Nov 2016 10:53:52 -0800 (PST)
Subject: [pypy-commit] pypy default: workaround
Message-ID: <5839da40.c9b3c20a.40d2f.0b46@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88677:1d977ac6f362
Date: 2016-11-26 19:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1d977ac6f362/

Log:	workaround

diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -204,6 +204,10 @@
         pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
+
+    # XXX for Darwin running old versions of CPython 2.7.x
+    sysconfig.get_config_vars()
+
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler)  # XXX
     objects = []

From pypy.commits at gmail.com  Sat Nov 26 15:10:48 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 26 Nov 2016 12:10:48 -0800 (PST)
Subject: [pypy-commit] pypy default: Complain early if we attempt a
 non-incminimark translation with the cpyext module
Message-ID: <5839ec48.ca10c20a.5aae6.26d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88678:a18ed4af9a2d
Date: 2016-11-26 21:10 +0100
http://bitbucket.org/pypy/pypy/changeset/a18ed4af9a2d/

Log:	Complain early if we attempt a non-incminimark translation with the
	cpyext module

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -298,6 +298,12 @@
         if config.translation.sandbox:
             config.objspace.lonepycfiles = False
 
+        if config.objspace.usemodules.cpyext:
+            if config.translation.gc != 'incminimark':
+                raise Exception("The 'cpyext' module requires the 'incminimark'"
+                                " GC.  You need either 'targetpypystandalone.py"
+                                " --withoutmod-cpyext' or '--gc=incminimark'")
+
         config.translating = True
 
         import translate

From pypy.commits at gmail.com  Sat Nov 26 16:19:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Sat, 26 Nov 2016 13:19:43 -0800 (PST)
Subject: [pypy-commit] pypy default: OS/X fixes
Message-ID: <5839fc6f.849c1c0a.38a6c.0eb0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88679:38eefb6ad69d
Date: 2016-11-26 22:18 +0100
http://bitbucket.org/pypy/pypy/changeset/38eefb6ad69d/

Log:	OS/X fixes

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -74,6 +74,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
@@ -109,6 +122,12 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
     if args:
         vals = []
         for name in args:

From pypy.commits at gmail.com  Sun Nov 27 03:38:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 27 Nov 2016 00:38:25 -0800 (PST)
Subject: [pypy-commit] cffi default: Merged in
 umireon/cffi/build-with-xcode-only (pull request #73)
Message-ID: <583a9b81.448e1c0a.79cea.a32e@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r2821:29624c75b419
Date: 2016-11-27 08:38 +0000
http://bitbucket.org/cffi/cffi/changeset/29624c75b419/

Log:	Merged in umireon/cffi/build-with-xcode-only (pull request #73)

	Able to build on macOS only with Xcode but without CLT

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -122,6 +122,20 @@
 if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')
 
+if 'darwin' in sys.platform:
+    try:
+        p = subprocess.Popen(['xcrun', '--show-sdk-path'],
+                             stdout=subprocess.PIPE)
+    except OSError as e:
+        if e.errno not in [errno.ENOENT, errno.EACCES]:
+            raise
+    else:
+        t = p.stdout.read().decode().strip()
+        p.stdout.close()
+        if p.wait() == 0:
+            include_dirs.append(t + '/usr/include/ffi')
+
+
 
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension

From pypy.commits at gmail.com  Sun Nov 27 03:38:28 2016
From: pypy.commits at gmail.com (umireon)
Date: Sun, 27 Nov 2016 00:38:28 -0800 (PST)
Subject: [pypy-commit] cffi build-with-xcode-only: Able to build on macOS
 only with Xcode but without CLT
Message-ID: <583a9b84.2a0bc30a.71fd5.5fe3@mx.google.com>

Author: Kaito Udagawa <umireon at gmail.com>
Branch: build-with-xcode-only
Changeset: r2820:75a9bf95e46d
Date: 2016-11-26 00:35 +0900
http://bitbucket.org/cffi/cffi/changeset/75a9bf95e46d/

Log:	Able to build on macOS only with Xcode but without CLT

	In macOS, two different build environment exists: Xcode.app and
	Xcode Command Line Tools (aka CLT)

	By this change setuptools considers the header under Xcode.app, and
	cffi can build every environment where cc exists.

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -122,6 +122,20 @@
 if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')
 
+if 'darwin' in sys.platform:
+    try:
+        p = subprocess.Popen(['xcrun', '--show-sdk-path'],
+                             stdout=subprocess.PIPE)
+    except OSError as e:
+        if e.errno not in [errno.ENOENT, errno.EACCES]:
+            raise
+    else:
+        t = p.stdout.read().decode().strip()
+        p.stdout.close()
+        if p.wait() == 0:
+            include_dirs.append(t + '/usr/include/ffi')
+
+
 
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension

From pypy.commits at gmail.com  Sun Nov 27 04:02:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 27 Nov 2016 01:02:51 -0800 (PST)
Subject: [pypy-commit] pypy default: fix test_lloperation
Message-ID: <583aa13b.c5371c0a.a9d9e.9ff2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88680:32ed555877b1
Date: 2016-11-27 10:02 +0100
http://bitbucket.org/pypy/pypy/changeset/32ed555877b1/

Log:	fix test_lloperation

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -959,6 +959,9 @@
     def op_gc_rawrefcount_create_link_pypy(self, *args):
         raise NotImplementedError("gc_rawrefcount_create_link_pypy")
 
+    def op_gc_rawrefcount_mark_deallocating(self, *args):
+        raise NotImplementedError("gc_rawrefcount_mark_deallocating")
+
     def op_do_malloc_fixedsize(self):
         raise NotImplementedError("do_malloc_fixedsize")
     def op_do_malloc_fixedsize_clear(self):

From pypy.commits at gmail.com  Sun Nov 27 13:31:45 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 10:31:45 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: Create an assert_() function
 that can exactly replace asserts in RPython and doesn't trigger pytest's
 assertion rewriting
Message-ID: <583b2691.c89cc20a.e6962.7f77@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88681:da4b23a03b40
Date: 2016-11-27 18:28 +0000
http://bitbucket.org/pypy/pypy/changeset/da4b23a03b40/

Log:	Create an assert_() function that can exactly replace asserts in
	RPython and doesn't trigger pytest's assertion rewriting

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.rlib import objectmodel
+from rpython.rlib.objectmodel import assert_
 from rpython.flowspace.flowcontext import FlowingError
 from rpython.flowspace.operation import op
 
@@ -858,7 +859,7 @@
         class C(B):
             pass
         def f(x):
-            assert type(x) is C
+            assert_(type(x) is C)
             return x
         a = self.RPythonAnnotator()
         s = a.build_types(f, [B])
@@ -894,10 +895,10 @@
         assert isinstance(s, annmodel.SomeString)
 
     def test_ann_assert(self):
-        def assert_(x):
-            assert x,"XXX"
-        a = self.RPythonAnnotator()
-        s = a.build_types(assert_, [int])
+        def my_assert(x):
+            assert_(x, "XXX")
+        a = self.RPythonAnnotator()
+        s = a.build_types(my_assert, [int])
         assert s.const is None
 
     def test_string_and_none(self):
@@ -1059,7 +1060,7 @@
             if a:
                 c = a
             else:
-                assert b >= 0
+                assert_(b >= 0)
                 c = b
             return c
         a = self.RPythonAnnotator()
@@ -1214,7 +1215,7 @@
 
         def alloc(cls):
             i = inst(cls)
-            assert isinstance(i, cls)
+            assert_(isinstance(i, cls))
             return i
         alloc._annspecialcase_ = "specialize:arg(0)"
 
@@ -1250,9 +1251,9 @@
 
         def alloc(cls, cls2):
             i = cls()
-            assert isinstance(i, cls)
+            assert_(isinstance(i, cls))
             j = cls2()
-            assert isinstance(j, cls2)
+            assert_(isinstance(j, cls2))
             return i
 
         def f():
@@ -1360,7 +1361,7 @@
         class T(object):
             pass
         def g(l):
-            assert isinstance(l, list)
+            assert_(isinstance(l, list))
             return l
         def f():
             l = [T()]
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -759,7 +759,6 @@
         w_value = self.peekvalue()
         if self.guessbool(op.bool(w_value).eval(self)):
             return target
-            return target
         self.popvalue()
 
     def JUMP_IF_NOT_DEBUG(self, target):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -14,7 +14,7 @@
 from rpython.tool.sourcetools import rpython_wrapper, func_with_new_name
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.flowspace.specialcase import register_flow_sc
-from rpython.flowspace.model import Constant
+from rpython.flowspace.model import Constant, const
 
 # specialize is a decorator factory for attaching _annspecialcase_
 # attributes to functions: for example
@@ -230,6 +230,34 @@
     func._not_rpython_ = True
     return func
 
+def assert_(condition, msg=None):
+    """
+    A function version of the assert statement.
+
+    The RPython toolchain interprets it in exactly the same way as the
+    statement version. This is useful in tests to prevent py.test assertion
+    rewriting from modifying the bytecode of RPython functions.
+    """
+    if msg is None:
+        assert condition
+    else:
+        assert condition, msg
+
+ at register_flow_sc(assert_)
+def sc_assert(ctx, *args_w):
+    from rpython.flowspace.operation import op
+    from rpython.flowspace.flowcontext import Raise, FlowingError
+    if len(args_w) == 2:
+        w_condition, w_msg = args_w
+    elif len(args_w) == 1:
+        w_condition, = args_w
+        w_msg = const(None)
+    else:
+        raise FlowingError(
+            "assert_() requires 1 or 2 arguments, %s given" % len(args_w))
+    if not ctx.guessbool(op.bool(w_condition).eval(ctx)):
+        raise Raise(ctx.exc_from_raise(const(AssertionError), w_msg))
+
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Sun Nov 27 13:31:47 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 10:31:47 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: Stop forcing assert
 reinterpretation in pytest (i.e. use assert rewriting by default)
Message-ID: <583b2693.0b561c0a.d1ca8.55bb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88682:154038131b4c
Date: 2016-11-27 18:30 +0000
http://bitbucket.org/pypy/pypy/changeset/154038131b4c/

Log:	Stop forcing assert reinterpretation in pytest (i.e. use assert
	rewriting by default)

diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
--- a/pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-addopts = --assert=reinterp -rf

From pypy.commits at gmail.com  Sun Nov 27 15:15:51 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 12:15:51 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: Remove rpython/pytest.ini as well
Message-ID: <583b3ef7.c11d1c0a.72209.8839@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88683:9ba4ee5e7c4d
Date: 2016-11-27 19:55 +0000
http://bitbucket.org/pypy/pypy/changeset/9ba4ee5e7c4d/

Log:	Remove rpython/pytest.ini as well

diff --git a/rpython/pytest.ini b/rpython/pytest.ini
deleted file mode 100644
--- a/rpython/pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-addopts = --assert=reinterp -rf

From pypy.commits at gmail.com  Sun Nov 27 15:15:54 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 12:15:54 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: Better error reporting when
 seeing locals()
Message-ID: <583b3efa.0f341c0a.13702.8454@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88684:7f27f353a8f6
Date: 2016-11-27 20:14 +0000
http://bitbucket.org/pypy/pypy/changeset/7f27f353a8f6/

Log:	Better error reporting when seeing locals()

diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -32,13 +32,12 @@
 
 @register_flow_sc(locals)
 def sc_locals(_, *args):
-    raise Exception(
-        "A function calling locals() is not RPython.  "
-        "Note that if you're translating code outside the PyPy "
-        "repository, a likely cause is that py.test's --assert=rewrite "
-        "mode is getting in the way.  You should copy the file "
-        "pytest.ini from the root of the PyPy repository into your "
-        "own project.")
+    from rpython.flowspace.flowcontext import FlowingError
+    raise FlowingError(
+        "A function calling locals() is not RPython. A likely cause is that "
+        "py.test's assert rewriting is getting in the way. If so, you "
+        "should use `rpython.rlib.objectmodel.assert_()` instead of an "
+        "assert statement here.")
 
 @register_flow_sc(getattr)
 def sc_getattr(ctx, w_obj, w_index, w_default=None):
diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py
--- a/rpython/translator/c/test/test_genc.py
+++ b/rpython/translator/c/test/test_genc.py
@@ -8,7 +8,7 @@
 from rpython.rlib.entrypoint import entrypoint_highlevel
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, assert_
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem.rstr import STR
@@ -78,7 +78,7 @@
                 if a == 'True':
                     args += (True,)
                 else:
-                    assert a == 'False'
+                    assert_(a == 'False')
                     args += (False,)
             elif argtype is float:
                 if a == 'inf':

From pypy.commits at gmail.com  Sun Nov 27 16:38:42 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 27 Nov 2016 13:38:42 -0800 (PST)
Subject: [pypy-commit] pypy default: Force right-shifts to be done on 64-bit
 values: otherwise, translating
Message-ID: <583b5262.08301c0a.670e7.8cef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88685:a86ded88180d
Date: 2016-11-27 22:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a86ded88180d/

Log:	Force right-shifts to be done on 64-bit values: otherwise,
	translating on 32-bit in lldebug mode crashes (and, in non-debug
	mode, can produce undefined behavior).

diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -13,6 +13,7 @@
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.history import ConstInt, ConstFloat
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import r_longlong
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import compute_unique_id, always_inline
 from rpython.rlib.objectmodel import we_are_translated, specialize
@@ -113,6 +114,7 @@
 
 @always_inline
 def encode_le_64bit(val):
+    val = r_longlong(val)     # force 64-bit, even on 32-bit
     return ''.join([chr((val >> 0) & 0xff),
                     chr((val >> 8) & 0xff),
                     chr((val >> 16) & 0xff),

From pypy.commits at gmail.com  Sun Nov 27 17:48:22 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 27 Nov 2016 14:48:22 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2438: Windows: the
 QueryPerformanceCounter() function is a stdcall
Message-ID: <583b62b6.ca10c20a.5aae6.da15@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88686:558458473028
Date: 2016-11-27 23:47 +0100
http://bitbucket.org/pypy/pypy/changeset/558458473028/

Log:	Issue #2438: Windows: the QueryPerformanceCounter() function is a
	stdcall function with releasegil=False and no save_err. So
	rffi.llexternal() would not create any wrapper, and the JIT would
	directly call it. But it can't do that: wrong calling convention!

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -205,11 +205,17 @@
     else:
         # if we don't have to invoke the GIL handling, we can just call
         # the low-level function pointer carelessly
-        if macro is None and save_err == RFFI_ERR_NONE:
+        # ...well, unless it's a macro, in which case we still have
+        # to hide it from the JIT...
+        need_wrapper = (macro is not None or save_err != RFFI_ERR_NONE)
+        # XXX ...and unless we're on Windows, because the calling convention
+        #     is unknown so far and thus the JIT can't assume it knows it...
+        if sys.platform == 'win32':
+            need_wrapper = True
+        #
+        if not need_wrapper:
             call_external_function = funcptr
         else:
-            # ...well, unless it's a macro, in which case we still have
-            # to hide it from the JIT...
             argnames = ', '.join(['a%d' % i for i in range(len(args))])
             source = py.code.Source("""
                 def call_external_function(%(argnames)s):

From pypy.commits at gmail.com  Sun Nov 27 19:11:26 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 16:11:26 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: Add back empty pytest.ini,
 as buildbot seems to need it
Message-ID: <583b762e.12ad1c0a.4a861.c2d5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88688:ea0a4f2e94e1
Date: 2016-11-28 00:01 +0000
http://bitbucket.org/pypy/pypy/changeset/ea0a4f2e94e1/

Log:	Add back empty pytest.ini, as buildbot seems to need it

diff --git a/pytest.ini b/pytest.ini
new file mode 100644

From pypy.commits at gmail.com  Sun Nov 27 19:11:24 2016
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 27 Nov 2016 16:11:24 -0800 (PST)
Subject: [pypy-commit] pypy assert-rewrite: fix asserts in rpython/annotator/
Message-ID: <583b762c.4673c20a.e67f3.f623@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: assert-rewrite
Changeset: r88687:0bb35d6c9f51
Date: 2016-11-27 23:50 +0000
http://bitbucket.org/pypy/pypy/changeset/0bb35d6c9f51/

Log:	fix asserts in rpython/annotator/

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -1649,7 +1649,7 @@
 
     def test_more_nonneg_cleverness(self):
         def f(start, stop):
-            assert 0 <= start <= stop
+            assert_(0 <= start <= stop)
             return start, stop
         a = self.RPythonAnnotator()
         s = a.build_types(f, [int, int])
@@ -1657,7 +1657,7 @@
 
     def test_more_general_nonneg_cleverness(self):
         def f(start, stop):
-            assert 0 <= start <= stop
+            assert_(0 <= start <= stop)
             return start, stop
         a = self.RPythonAnnotator()
         s = a.build_types(f, [r_longlong, r_longlong])
@@ -1829,7 +1829,7 @@
 
     def test_int_nonneg(self):
         def f(x, y):
-            assert x >= 0
+            assert_(x >= 0)
             return int(x) + int(y == 3)
         a = self.RPythonAnnotator()
         s = a.build_types(f, [int, int])
@@ -3375,7 +3375,7 @@
         def g():
             should_not_see_this
         def f(n):
-            assert n >= 0
+            assert_(n >= 0)
             if n < 0:
                 g()
             if not (n >= 0):
@@ -4244,7 +4244,7 @@
 
         class B(A):
             def __init__(self, x):
-                assert x >= 0
+                assert_(x >= 0)
                 self.x = x
 
         def fn(i):

From pypy.commits at gmail.com  Sun Nov 27 19:44:51 2016
From: pypy.commits at gmail.com (arigo)
Date: Sun, 27 Nov 2016 16:44:51 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2438: Deprecate 'calling_conv',
 because it's too easy to get
Message-ID: <583b7e03.96a61c0a.e4f42.ccb5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88689:cb398f434016
Date: 2016-11-28 00:52 +0100
http://bitbucket.org/pypy/pypy/changeset/cb398f434016/

Log:	Issue #2438: Deprecate 'calling_conv', because it's too easy to get
	wrong (e.g. in QueryPerformanceCounter) and it's usually not fatal
	expect in the JIT after a lot of assembly-level debugging in visual
	studio.

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -7,7 +7,7 @@
 from rpython.rtyper.lltypesystem.llmemory import itemoffsetof
 from rpython.rtyper.llannotation import lltype_to_annotation
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, specialize
 from rpython.rlib.objectmodel import keepalive_until_here, enforceargs
 from rpython.rlib import rarithmetic, rgc
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -96,6 +96,9 @@
                 we consider that the function is really short-running and
                 don't bother releasing the GIL.  An explicit True or False
                 overrides this logic.
+
+    calling_conv: deprecated, because it's hard to get it right 100% of the
+                  time.  Nowadays it is ignored except for tests.
     """
     if _callable is not None:
         assert callable(_callable)
@@ -314,10 +317,6 @@
     # for debugging, stick ll func ptr to that
     wrapper._ptr = funcptr
     wrapper = func_with_new_name(wrapper, name)
-
-    if calling_conv != "c":
-        wrapper = jit.dont_look_inside(wrapper)
-
     return wrapper
 
 
@@ -1291,13 +1290,23 @@
 
 # You would have to have a *huge* amount of data for this to block long enough
 # to be worth it to release the GIL.
-c_memcpy = llexternal("memcpy",
+_c_memcpy = llexternal("memcpy",
             [VOIDP, VOIDP, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            _nowrapper=True
         )
-c_memset = llexternal("memset",
+_c_memset = llexternal("memset",
             [VOIDP, lltype.Signed, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            _nowrapper=True
         )
+
+ at specialize.ll()
+def c_memcpy(dst, src, size):
+    _c_memcpy(cast(VOIDP, dst), cast(VOIDP, src), cast(SIZE_T, size))
+
+ at specialize.ll()
+def c_memset(s, c, n):
+    _c_memset(cast(VOIDP, s), cast(lltype.Signed, c), cast(SIZE_T, n))

From pypy.commits at gmail.com  Mon Nov 28 04:02:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 01:02:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Issue #2438: Windows: llexternal
 functions declared explicitly with
Message-ID: <583bf28d.d39a1c0a.82b0d.b341@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88690:047b28974149
Date: 2016-11-28 09:58 +0100
http://bitbucket.org/pypy/pypy/changeset/047b28974149/

Log:	Issue #2438: Windows: llexternal functions declared explicitly with
	calling_conv='c' should still be directly callable by the JIT
	(performance only)

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -74,7 +74,8 @@
 def llexternal(name, args, result, _callable=None,
                compilation_info=ExternalCompilationInfo(),
                sandboxsafe=False, releasegil='auto',
-               _nowrapper=False, calling_conv='c',
+               _nowrapper=False,
+               calling_conv='unknown' if sys.platform == 'win32' else 'c',
                elidable_function=False, macro=None,
                random_effects_on_gcobjs='auto',
                save_err=RFFI_ERR_NONE):
@@ -97,8 +98,9 @@
                 don't bother releasing the GIL.  An explicit True or False
                 overrides this logic.
 
-    calling_conv: deprecated, because it's hard to get it right 100% of the
-                  time.  Nowadays it is ignored except for tests.
+    calling_conv: if 'unknown' or 'win', the C function is not directly seen
+                  by the JIT.  If 'c', it can be seen (depending on
+                  releasegil=False).  For tests only, it defaults to 'c'.
     """
     if _callable is not None:
         assert callable(_callable)
@@ -110,7 +112,8 @@
             _callable = generate_macro_wrapper(
                 name, macro, ext_type, compilation_info)
         else:
-            _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv)
+            _callable = ll2ctypes.LL2CtypesCallable(ext_type,
+                'c' if calling_conv == 'unknown' else calling_conv)
     else:
         assert macro is None, "'macro' is useless if you specify '_callable'"
     if elidable_function:
@@ -211,9 +214,9 @@
         # ...well, unless it's a macro, in which case we still have
         # to hide it from the JIT...
         need_wrapper = (macro is not None or save_err != RFFI_ERR_NONE)
-        # XXX ...and unless we're on Windows, because the calling convention
-        #     is unknown so far and thus the JIT can't assume it knows it...
-        if sys.platform == 'win32':
+        # ...and unless we're on Windows and the calling convention is
+        # 'win' or 'unknown'
+        if calling_conv != 'c':
             need_wrapper = True
         #
         if not need_wrapper:
@@ -1290,23 +1293,15 @@
 
 # You would have to have a *huge* amount of data for this to block long enough
 # to be worth it to release the GIL.
-_c_memcpy = llexternal("memcpy",
+c_memcpy = llexternal("memcpy",
             [VOIDP, VOIDP, SIZE_T],
             lltype.Void,
             releasegil=False,
-            _nowrapper=True
+            calling_conv='c',
         )
-_c_memset = llexternal("memset",
+c_memset = llexternal("memset",
             [VOIDP, lltype.Signed, SIZE_T],
             lltype.Void,
             releasegil=False,
-            _nowrapper=True
+            calling_conv='c',
         )
-
- at specialize.ll()
-def c_memcpy(dst, src, size):
-    _c_memcpy(cast(VOIDP, dst), cast(VOIDP, src), cast(SIZE_T, size))
-
- at specialize.ll()
-def c_memset(s, c, n):
-    _c_memset(cast(VOIDP, s), cast(lltype.Signed, c), cast(SIZE_T, n))

From pypy.commits at gmail.com  Mon Nov 28 04:12:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 01:12:11 -0800 (PST)
Subject: [pypy-commit] cffi default: merge heads
Message-ID: <583bf4eb.06891c0a.13802.4ecc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2823:9deaf0080a20
Date: 2016-11-28 10:11 +0100
http://bitbucket.org/cffi/cffi/changeset/9deaf0080a20/

Log:	merge heads

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -122,6 +122,20 @@
 if 'freebsd' in sys.platform:
     include_dirs.append('/usr/local/include')
 
+if 'darwin' in sys.platform:
+    try:
+        p = subprocess.Popen(['xcrun', '--show-sdk-path'],
+                             stdout=subprocess.PIPE)
+    except OSError as e:
+        if e.errno not in [errno.ENOENT, errno.EACCES]:
+            raise
+    else:
+        t = p.stdout.read().decode().strip()
+        p.stdout.close()
+        if p.wait() == 0:
+            include_dirs.append(t + '/usr/include/ffi')
+
+
 
 if __name__ == '__main__':
     from setuptools import setup, Distribution, Extension

From pypy.commits at gmail.com  Mon Nov 28 04:12:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 01:12:09 -0800 (PST)
Subject: [pypy-commit] cffi default: Document should_clear_after_alloc a bit
 more
Message-ID: <583bf4e9.2a0bc30a.71fd5.0ca3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2822:0782c203b090
Date: 2016-11-28 10:11 +0100
http://bitbucket.org/cffi/cffi/changeset/0782c203b090/

Log:	Document should_clear_after_alloc a bit more

diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -44,6 +44,11 @@
 used for a longer time.  Be careful about that when copying the
 pointer to the memory somewhere else, e.g. into another structure.
 
+The returned memory is initially cleared (filled with zeroes), before
+the optional initializer is applied.  For performance, see
+`ffi.new_allocator()`_ for a way to allocate non-zero-initialized
+memory.
+
 
 ffi.cast()
 ++++++++++
@@ -471,7 +476,16 @@
 
 If ``should_clear_after_alloc`` is set to False, then the memory
 returned by ``alloc()`` is assumed to be already cleared (or you are
-fine with garbage); otherwise CFFI will clear it.
+fine with garbage); otherwise CFFI will clear it.  Example: for
+performance, if you are using ``ffi.new()`` to allocate large chunks of
+memory where the initial content can be left uninitialized, you can do::
+
+    # at module level
+    new_nonzero = ffi.new_allocator(should_clear_after_alloc)
+
+    # then replace `p = ffi.new("char[]", bigsize)` with:
+        p = new_nonzero("char[]", bigsize)
+        
 
 
 ffi.init_once()

From pypy.commits at gmail.com  Mon Nov 28 04:55:49 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 01:55:49 -0800 (PST)
Subject: [pypy-commit] pypy default: _nowrapper functions now need an
 explicit calling_conv in order to be JIT-callable
Message-ID: <583bff25.0b561c0a.d1ca8.5646@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88691:5557b759580a
Date: 2016-11-28 10:55 +0100
http://bitbucket.org/pypy/pypy/changeset/5557b759580a/

Log:	_nowrapper functions now need an explicit calling_conv in order to
	be JIT-callable

diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -69,12 +69,14 @@
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    calling_conv='c')
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True)
+    _nowrapper=True, elidable_function=True, sandboxsafe=True,
+    calling_conv='c')
 
 
 class Float2LongLongEntry(ExtRegistryEntry):

From pypy.commits at gmail.com  Mon Nov 28 05:23:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 02:23:18 -0800 (PST)
Subject: [pypy-commit] pypy default: Change the default again: if
 _nowrapper=True, then we assume the
Message-ID: <583c0596.c89cc20a.e6962.8d2c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88692:52e737c73e60
Date: 2016-11-28 11:20 +0100
http://bitbucket.org/pypy/pypy/changeset/52e737c73e60/

Log:	Change the default again: if _nowrapper=True, then we assume the
	default calling convention of 'c'. Looks reasonable because
	_nowrapper should be used mostly for internal RPython helpers
	written in C. A review of all usages seems to confirm this.

diff --git a/rpython/rlib/longlong2float.py b/rpython/rlib/longlong2float.py
--- a/rpython/rlib/longlong2float.py
+++ b/rpython/rlib/longlong2float.py
@@ -69,14 +69,12 @@
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True,
-    calling_conv='c')
+    _nowrapper=True, elidable_function=True, sandboxsafe=True)
 
 singlefloat2uint = rffi.llexternal(
     "pypy__singlefloat2uint", [rffi.FLOAT], rffi.UINT,
     _callable=singlefloat2uint_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True,
-    calling_conv='c')
+    _nowrapper=True, elidable_function=True, sandboxsafe=True)
 
 
 class Float2LongLongEntry(ExtRegistryEntry):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -74,8 +74,7 @@
 def llexternal(name, args, result, _callable=None,
                compilation_info=ExternalCompilationInfo(),
                sandboxsafe=False, releasegil='auto',
-               _nowrapper=False,
-               calling_conv='unknown' if sys.platform == 'win32' else 'c',
+               _nowrapper=False, calling_conv=None,
                elidable_function=False, macro=None,
                random_effects_on_gcobjs='auto',
                save_err=RFFI_ERR_NONE):
@@ -100,8 +99,14 @@
 
     calling_conv: if 'unknown' or 'win', the C function is not directly seen
                   by the JIT.  If 'c', it can be seen (depending on
-                  releasegil=False).  For tests only, it defaults to 'c'.
+                  releasegil=False).  For tests only, or if _nowrapper,
+                  it defaults to 'c'.
     """
+    if calling_conv is None:
+        if sys.platform == 'win32' and not _nowrapper:
+            calling_conv = 'unknown'
+        else:
+            calling_conv = 'c'
     if _callable is not None:
         assert callable(_callable)
     ext_type = lltype.FuncType(args, result)

From pypy.commits at gmail.com  Mon Nov 28 05:23:57 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 02:23:57 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-multiprocessing: remove skip in conftest
 for two multiprocessing test suites (they do not deadlock anymore)
Message-ID: <583c05bd.2a0bc30a.71fd5.2cd7@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-multiprocessing
Changeset: r88693:cfe2cd888582
Date: 2016-11-21 12:50 +0100
http://bitbucket.org/pypy/pypy/changeset/cfe2cd888582/

Log:	remove skip in conftest for two multiprocessing test suites (they do
	not deadlock anymore)

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -303,9 +303,9 @@
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py'),
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
-    RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_fork.py'),
     RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
-    RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
+    RegrTest('test_multiprocessing_main_handling.py'),
     RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
     RegrTest('test_netrc.py'),
     RegrTest('test_nis.py'),

From pypy.commits at gmail.com  Mon Nov 28 05:23:59 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 02:23:59 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-multiprocessing: use memoryview in python3,
 not buffer
Message-ID: <583c05bf.876ec20a.c65b4.9d4d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-multiprocessing
Changeset: r88694:489f39369c1d
Date: 2016-11-21 15:42 +0100
http://bitbucket.org/pypy/pypy/changeset/489f39369c1d/

Log:	use memoryview in python3, not buffer

diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -85,7 +85,7 @@
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
-        buf = buffer(obj, offset, size)
+        buf = memoryview(obj[offset:offset+size])
         if len(buf) < size:
             raise ValueError(
                 "Buffer size too small (%d instead of at least %d bytes)"
@@ -97,7 +97,8 @@
 
     def from_buffer_copy(self, obj, offset=0):
         size = self._sizeofinstances()
-        buf = buffer(obj, offset, size)
+        buf = memoryview(obj, offset, size)
+        buf = memoryview(obj[offset:offset+size])
         if len(buf) < size:
             raise ValueError(
                 "Buffer size too small (%d instead of at least %d bytes)"

From pypy.commits at gmail.com  Mon Nov 28 05:24:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 02:24:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: catch up with py3.5
Message-ID: <583c05c2.c220c20a.7e1c1.a033@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88695:2294debaacb0
Date: 2016-11-28 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/2294debaacb0/

Log:	catch up with py3.5

diff too long, truncating to 2000 out of 3598 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,18 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -118,30 +124,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -370,12 +370,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -428,15 +429,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -12,7 +12,7 @@
 
 import sys
 import os
-import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -61,11 +61,19 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['SHLIB_SUFFIX'] = g['SO']
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -76,7 +84,7 @@
     g = {}
     g['EXE'] = ".exe"
     g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = g['SO'].rsplit('.')[0]   # xxx?
 
     global _config_vars
     _config_vars = g
@@ -118,26 +126,78 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+        (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = shlib_suffix
 
 
 from .sysconfig_cpython import (
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -29,3 +29,8 @@
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt
+
+.. branch: conditional_call_value_4
+
+Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
+call this function" returning a result.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -119,7 +119,7 @@
         try:
             # initialize sys.{path,executable,stdin,stdout,stderr}
             # (in unbuffered mode, to avoid troubles) and import site
-            space.appexec([w_path, space.wrap(home), w_initstdio],
+            space.appexec([w_path, space.wrap_fsdecoded(home), w_initstdio],
             r"""(path, home, initstdio):
                 import sys
                 sys.path[:] = path
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -113,7 +113,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -127,7 +129,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -396,7 +400,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -417,7 +421,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -20,7 +20,7 @@
     If the user has set this warning to raise an error, a SyntaxError will be
     raised."""
     w_msg = space.wrap(msg)
-    w_filename = space.wrap(fn)
+    w_filename = space.wrap_fsdecoded(fn)
     w_lineno = space.wrap(lineno)
     w_offset = space.wrap(offset)
     _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset)
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -30,7 +30,7 @@
                                            'replace')
             w_text = space.wrap(text)
         if self.filename is not None:
-            w_filename = space.fsdecode(space.newbytes(self.filename))
+            w_filename = space.wrap_fsdecoded(self.filename)
         return space.newtuple([space.wrap(self.msg),
                                space.newtuple([w_filename,
                                                space.wrap(self.lineno),
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -127,6 +127,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -886,3 +892,19 @@
         assert "keywords must be strings" in str(e.value)
         e = raises(TypeError, "f(y=2, **{'x': 5}, x=6)")
         assert "got multiple values for keyword argument 'x'" in str(e.value)
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -62,12 +62,14 @@
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
+    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.ensure_object_strategy()  # make sure we can return a borrowed obj
-    # XXX ^^^ how does this interact with CPyListStrategy?
+    cpy_strategy = space.fromcache(CPyListStrategy)
+    if w_list.strategy is not cpy_strategy:
+        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
     w_res = w_list.getitem(index)
     return w_res     # borrowed ref
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -112,12 +112,10 @@
     The memoryview object then owns the buffer represented by view, which
     means you shouldn't try to call PyBuffer_Release() yourself: it
     will be done on deallocation of the memoryview object."""
-    if not view.c_buf:
-        raise oefmt(space.w_ValueError,
-                    "cannot make memory view from a buffer with a NULL data "
-                    "pointer")
-    buf = CBuffer(space, view.c_buf, view.c_len, view.c_obj)
-    return space.wrap(W_MemoryView(buf))
+    w_obj = from_ref(space, view.c_obj)
+    if isinstance(w_obj, W_MemoryView):
+        return w_obj
+    return space.call_method(space.builtin, "memoryview", w_obj)
 
 @cpython_api([PyObject], PyObject)
 def PyMemoryView_GET_BASE(space, w_obj):
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -111,11 +111,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
+    return _PyUnicode_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -260,7 +260,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.wrap('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_ASCII(space, w_obj):
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -4,15 +4,6 @@
 class AppTestArrayModule(AppTestCpythonExtensionBase):
     enable_leak_checking = True
 
-    def setup_class(cls):
-        from rpython.tool.udir import udir
-        AppTestCpythonExtensionBase.setup_class.im_func(cls)
-        if option.runappdirect:
-            cls.w_udir = str(udir)
-        else:
-            cls.w_udir = cls.space.wrap(str(udir))
-
-
     def test_basic(self):
         module = self.import_module(name='array')
         arr = module.array('i', [1,2,3])
@@ -105,30 +96,28 @@
         # Not really part of array, refactor
         import struct
         module = self.import_module(name='array')
-        val = module.readbuffer_as_string('abcd')
-        assert val == 'abcd'
-        val = module.readbuffer_as_string(u'\u03a3')
-        assert val is not None
+        val = module.readbuffer_as_string(b'abcd')
+        assert val == b'abcd'
 
     def test_readinto(self):
         module = self.import_module(name='array')
-        a = module.array('c')
-        a.fromstring('0123456789')
+        a = module.array('B')
+        a.fromstring(b'0123456789')
         filename = self.udir + "/_test_file"
         f = open(filename, 'w+b')
-        f.write('foobar')
+        f.write(b'foobar')
         f.seek(0)
         n = f.readinto(a)
         f.close()
         assert n == 6
         assert len(a) == 10
-        assert a.tostring() == 'foobar6789'
+        assert a.tostring() == b'foobar6789'
 
     def test_iowrite(self):
         module = self.import_module(name='array')
         from io import BytesIO
-        a = module.array('c')
-        a.fromstring('0123456789')
+        a = module.array('B')
+        a.fromstring(b'0123456789')
         fd = BytesIO()
         # only test that it works
         fd.write(a)
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -203,3 +214,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -285,6 +285,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -106,7 +106,7 @@
 
 class AppTestFetch(AppTestCpythonExtensionBase):
     def setup_class(cls):
-        from pypy.module.imp.test.support import get_special_char
+        from pypy.interpreter.test.test_fsencode import get_special_char
         space = cls.space
         cls.special_char = get_special_char()
         cls.w_special_char = space.wrap(cls.special_char)
@@ -380,7 +380,7 @@
         "XXX seems to pass, but doesn't: 'py.test -s' shows errors in PyObject_Free")
     def test_GetSetExcInfo(self):
         import sys
-        if self.runappdirect and (sys.version_info.major < 3 or 
+        if self.runappdirect and (sys.version_info.major < 3 or
                                   sys.version_info.minor < 3):
             skip('PyErr_{GS}etExcInfo introduced in python 3.3')
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -306,12 +306,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -467,7 +467,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -725,14 +725,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
         if not py_type.c_tp_as_sequence:
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -75,6 +75,9 @@
              abs_sys    = "import sys",
              struct     = "inpackage = 1",
              errno      = "",
+             # Python 3 note: this __future__ has no effect any more,
+             # kept around for testing and to avoid increasing the diff
+             # with PyPy2
              absolute   = "from __future__ import absolute_import\nimport struct",
              relative_b = "from __future__ import absolute_import\nfrom . import struct",
              relative_c = "from __future__ import absolute_import\nfrom .struct import inpackage",
@@ -597,7 +600,7 @@
         imp.reload(sys)
 
         assert sys.path is oldpath
-        assert 'settrace' in dir(sys)
+        assert 'settrace' not in dir(sys)    # at least on CPython 3.5.2
 
     def test_reload_builtin_doesnt_clear(self):
         import imp
@@ -657,16 +660,15 @@
             assert False, 'should not work'
 
     def test_cache_from_source(self):
-        import imp
+        import imp, sys
+        tag = sys.implementation.cache_tag
         pycfile = imp.cache_from_source('a/b/c.py')
-        assert pycfile.startswith('a/b/__pycache__/c.pypy3-')
-        assert pycfile.endswith('.pyc')
-        assert imp.source_from_cache('a/b/__pycache__/c.pypy3-17.pyc'
+        assert pycfile == 'a/b/__pycache__/c.%s.pyc' % tag
+        assert imp.source_from_cache('a/b/__pycache__/c.%s.pyc' % tag
                                      ) == 'a/b/c.py'
         raises(ValueError, imp.source_from_cache, 'a/b/c.py')
 
     def test_invalid_pathname(self):
-        skip("This test fails on CPython 3.3, but passes on CPython 3.4+")
         import imp
         import pkg
         import os
@@ -906,6 +908,10 @@
             stream.close()
 
     def test_pyc_magic_changes(self):
+        # skipped: for now, PyPy generates only one kind of .pyc file
+        # per version.  Different versions should differ in
+        # sys.implementation.cache_tag, which means that they'll look up
+        # different .pyc files anyway.  See test_get_tag() in test_app.py.
         py.test.skip("For now, PyPy generates only one kind of .pyc files")
         # test that the pyc files produced by a space are not reimportable
         # from another, if they differ in what opcodes they support
@@ -1009,7 +1015,9 @@
         sys.meta_path.insert(0, Importer())
         try:
             import math
-            assert len(tried_imports) == 1
+            # the above line may trigger extra imports, like _operator
+            # from app_math.py.  The first one should be 'math'.
+            assert len(tried_imports) >= 1
             package_name = '.'.join(__name__.split('.')[:-1])
             if package_name:
                 assert tried_imports[0][0] == package_name + ".math"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -16,6 +16,7 @@
         return not detect_simd_z()
     return True
 
+ at py.test.mark.skipif(True, reason='no _numpypy on pypy3')
 class TestMicroNumPy(BaseTestPyPyC):
 
     arith_comb = [('+','float','float', 4*3427,   3427, 1.0,3.0),
diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py
--- a/pypy/module/pypyjit/test_pypy_c/test_shift.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py
@@ -47,9 +47,9 @@
             res = 0
             a = 0
             while a < 300:
-                res1 = a/b     # ID: div
-                res2 = a/2     # ID: shift
-                res3 = a/11    # ID: mul
+                res1 = a // b     # ID: div
+                res2 = a // 2     # ID: shift
+                res3 = a // 11    # ID: mul
                 res += res1 + res2 + res3
                 a += 1
             return res
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -11,58 +11,48 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def unicode_iter(space):
     "Utility that returns the app-level descriptor str.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_unicode, '__iter__')
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    return space.lookup_in_type(space.w_dict, '__getitem__')
+
 
 def raiseattrerror(space, w_obj, w_name, w_descr=None):
     # space.repr always returns an encodable string.
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -339,12 +339,18 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -374,6 +374,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -671,6 +675,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 
@@ -700,4 +708,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -79,9 +79,10 @@
 
     def identifier_w(self, space):
         try:
-            # call the elidable function, with a jit.call_shortcut in case
-            # self._utf8 is already computed
-            identifier = g_identifier_w(self, space)
+            identifier = jit.conditional_call_elidable(
+                                self._utf8, g_encode_utf8, self._value)
+            if not jit.isconstant(self):
+                self._utf8 = identifier
         except UnicodeEncodeError:
             # bah, this is just to get an official app-level
             # UnicodeEncodeError
@@ -1273,15 +1274,9 @@
                                      allow_surrogates=allow_surrogates)
 
 @jit.elidable
- at jit.call_shortcut
-def g_identifier_w(self, space):
-    """This is a global function because of @jit.call_shortcut"""
-    identifier = self._utf8
-    if identifier is not None:
-        return identifier
-    identifier = self._value.encode('utf-8')
-    self._utf8 = identifier
-    return identifier
+def g_encode_utf8(value):
+    """This is a global function because of jit.conditional_call_value"""
+    return value.encode('utf-8')
 
 _repr_function, _ = make_unicode_escape_function(
     pass_printable=True, unicode_output=True, quotes=True, prefix='')
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -22,7 +22,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -50,6 +51,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -202,6 +206,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -348,6 +358,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -392,6 +404,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -482,6 +498,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -326,7 +327,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -789,6 +789,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -811,6 +815,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -663,6 +664,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return compute_hash(hlstr(s))
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return compute_hash(hlunicode(u))
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,9 +9,9 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
@@ -347,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -595,33 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
-        descr = op.getdescr()
-        if descr is None:
-            return
-        assert isinstance(descr, CallDescr)
-        effectinfo = descr.get_extra_info()
-        if effectinfo is None or effectinfo.call_shortcut is None:
-            return
-        if op.type == 'r':
-            cond_call_opnum = rop.COND_CALL_VALUE_R
-        elif op.type == 'i':
-            cond_call_opnum = rop.COND_CALL_VALUE_I
-        else:
-            return
-        cs = effectinfo.call_shortcut
-        ptr_box = op.getarg(1 + cs.argnum)
-        if cs.fielddescr is not None:
-            value_box = self.emit_getfield(ptr_box, descr=cs.fielddescr,
-                                           raw=(ptr_box.type == 'i'))
-        else:
-            value_box = ptr_box
-        self.replace_op_with(op, ResOperation(cond_call_opnum,
-                                              [value_box] + op.getarglist(),
-                                              descr=descr))
-
     def handle_call_assembler(self, op):
         descrs = self.gc_ll_descr.getframedescrs(self.cpu)
         loop_token = op.getdescr()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,8 +1,7 @@
 import py
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescr, get_interiorfield_descr, get_call_descr
-from rpython.jit.codewriter.effectinfo import EffectInfo, CallShortcut
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -81,21 +80,6 @@
                                      lltype.malloc(T, zero=True))
         self.myT = myT
         #
-        call_shortcut = CallShortcut(0, tzdescr)
-        effectinfo = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut)
-        call_shortcut_descr = get_call_descr(self.gc_ll_descr,
-            [lltype.Ptr(T)], lltype.Signed,
-            effectinfo)
-        call_shortcut_2 = CallShortcut(0, None)
-        effectinfo_2 = EffectInfo(None, None, None, None, None, None,
-                                EffectInfo.EF_RANDOM_EFFECTS,
-                                call_shortcut=call_shortcut_2)
-        call_shortcut_descr_2 = get_call_descr(self.gc_ll_descr,
-            [lltype.Signed], lltype.Signed,
-            effectinfo_2)
-        #
         A = lltype.GcArray(lltype.Signed)
         adescr = get_array_descr(self.gc_ll_descr, A)
         adescr.tid = 4321
@@ -216,7 +200,6 @@
 
     load_constant_offset = True
     load_supported_factors = (1,2,4,8)
-    supports_cond_call_value = True
 
     translate_support_code = None
 
@@ -1239,6 +1222,10 @@
                        'i3 = gc_load_i(p0,'
                                  '%(strlendescr.offset)s,'
                                  '%(strlendescr.field_size)s)'],
+        [True,  (1,),  'i3 = strhash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(strhashdescr.offset)s,'
+                                 '-%(strhashdescr.field_size)s)'],
         #[False, (1,),  'i3 = unicodelen(p0)' '->'
         #               'i3 = gc_load_i(p0,'
         #                       '%(unicodelendescr.offset)s,'
@@ -1247,7 +1234,10 @@
                        'i3 = gc_load_i(p0,'
                                '%(unicodelendescr.offset)s,'
                                '%(unicodelendescr.field_size)s)'],
-
+        [True,  (1,),  'i3 = unicodehash(p0)' '->'
+                       'i3 = gc_load_i(p0,'
+                                 '%(unicodehashdescr.offset)s,'
+                                 '-%(unicodehashdescr.field_size)s)'],
         ## getitem str/unicode
         [True,  (2,4), 'i3 = unicodegetitem(p0,i1)' '->'
                        'i3 = gc_load_indexed_i(p0,i1,'
@@ -1446,26 +1436,3 @@
             jump()
         """)
         assert len(self.gcrefs) == 2
-
-    def test_handle_call_shortcut(self):
-        self.check_rewrite("""
-            [p0]
-            i1 = call_i(123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """, """
-            [p0]
-            i2 = gc_load_i(p0, %(tzdescr.offset)s, %(tzdescr.field_size)s)
-            i1 = cond_call_value_i(i2, 123, p0, descr=call_shortcut_descr)
-            jump(i1)
-        """)
-
-    def test_handle_call_shortcut_2(self):
-        self.check_rewrite("""
-            [i0]
-            i1 = call_i(123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """, """
-            [i0]
-            i1 = cond_call_value_i(i0, 123, i0, descr=call_shortcut_descr_2)
-            jump(i1)
-        """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -16,7 +16,6 @@
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
     supports_guard_gc_type = False
-    supports_cond_call_value = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -656,10 +656,12 @@
     _COND_CALL_SAVE_REGS = [r.r3, r.r4, r.r5, r.r6, r.r12]
 
     def emit_cond_call(self, op, arglocs, regalloc):
+        resloc = arglocs[0]
+        arglocs = arglocs[1:]
+
         fcond = self.guard_success_cc
         self.guard_success_cc = c.cond_none
         assert fcond != c.cond_none
-        fcond = c.negate(fcond)
 
         jmp_adr = self.mc.get_relative_pos()
         self.mc.trap()        # patched later to a 'bc'
@@ -691,7 +693,10 @@
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.bl_abs(cond_call_adr)
         # restoring the registers saved above, and doing pop_gcmap(), is left
-        # to the cond_call_slowpath helper.  We never have any result value.
+        # to the cond_call_slowpath helper.  If we have a result, move
+        # it from r2 to its expected location.
+        if resloc is not None:
+            self.mc.mr(resloc.value, r.SCRATCH2.value)
         relative_target = self.mc.currpos() - jmp_adr
         pmc = OverwritingBuilder(self.mc, jmp_adr, 1)
         BI, BO = c.encoding[fcond]
@@ -701,6 +706,9 @@
         # guard_no_exception too
         self.previous_cond_call_jcond = jmp_adr, BI, BO
 
+    emit_cond_call_value_i = emit_cond_call
+    emit_cond_call_value_r = emit_cond_call
+
 
 class FieldOpAssembler(object):
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -315,6 +315,7 @@
         #   * r2 is the gcmap
         #   * the old value of these regs must already be stored in the jitframe
         #   * on exit, all registers are restored from the jitframe
+        #   * the result of the call, if any, is moved to r2
 
         mc = PPCBuilder()
         self.mc = mc
@@ -347,7 +348,11 @@
         # Finish
         self._reload_frame_if_necessary(mc)
 
+        # Move the result, if any, to r2
+        mc.mr(r.SCRATCH2.value, r.r3.value)
+
         mc.mtlr(r.RCS1.value)     # restore LR
+
         self._pop_core_regs_from_jitframe(mc, saved_regs)
         if supports_floats:
             self._pop_fp_regs_from_jitframe(mc)
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -1038,14 +1038,34 @@
 
     def prepare_cond_call(self, op):
         self.load_condition_into_cc(op.getarg(0))
-        locs = []
+        self.assembler.guard_success_cc = c.negate(
+            self.assembler.guard_success_cc)
+        # ^^^ if arg0==0, we jump over the next block of code (the call)
+        locs = [None]
         # support between 0 and 4 integer arguments
         assert 2 <= op.numargs() <= 2 + 4
         for i in range(1, op.numargs()):
             loc = self.loc(op.getarg(i))
             assert loc.type != FLOAT
             locs.append(loc)
-        return locs
+        return locs     # [None, function, args...]
+
+    def prepare_cond_call_value_i(self, op):
+        x = self.ensure_reg(op.getarg(0))
+        self.load_condition_into_cc(op.getarg(0))
+        self.rm.force_allocate_reg(op, selected_reg=x)   # spilled if survives
+        # ^^^ if arg0!=0, we jump over the next block of code (the call)
+        locs = [x]
+        # support between 0 and 4 integer arguments
+        assert 2 <= op.numargs() <= 2 + 4
+        for i in range(1, op.numargs()):
+            loc = self.loc(op.getarg(i))
+            assert loc.type != FLOAT
+            locs.append(loc)
+        return locs     # [res, function, args...]
+
+    prepare_cond_call_value_r = prepare_cond_call_value_i
+
 
 def notimplemented(self, op):
     msg = '[PPC/regalloc] %s not implemented\n' % op.getopname()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -2448,9 +2448,6 @@
             assert called == [(67, 89)]
 
     def test_cond_call_value(self):
-        if not self.cpu.supports_cond_call_value:
-            py.test.skip("missing supports_cond_call_value")
-
         def func_int(*args):
             called.append(args)
             return len(args) * 100 + 1000
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -711,11 +711,6 @@
 # 6. a conditional call (for now always with no exception raised)
 class CondCallOperation(BaseCallOperation):
 
-    def filter(self, builder):
-        if not builder.cpu.supports_cond_call_value and \
-           self.opnum == rop.COND_CALL_VALUE_I:
-            raise test_random.CannotProduceOperation
-
     def produce_into(self, builder, r):
         fail_subset = builder.subset_of_intvars(r)
         if self.opnum == rop.COND_CALL:
diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py
--- a/rpython/jit/backend/test/zll_stress.py
+++ b/rpython/jit/backend/test/zll_stress.py
@@ -20,11 +20,6 @@
     r = Random()
     r.jumpahead(piece*99999999)
     OPERATIONS = LLtypeOperationBuilder.OPERATIONS[:]
-    if not cpu.supports_cond_call_value:
-        # remove COND_CALL_VALUE_I if the cpu does not support it
-        ops = LLtypeOperationBuilder.OPERATIONS
-        LLtypeOperationBuilder.OPERATIONS = [op for op in ops \
-                if op.opnum != rop.COND_CALL_VALUE_I]
     for i in range(piece*per_piece, (piece+1)*per_piece):
         print "        i = %d; r.setstate(%s)" % (i, r.getstate())
         check_random_function(cpu, LLtypeOperationBuilder, r, i, total_iterations)
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -16,7 +16,6 @@
     debug = True
     supports_floats = True
     supports_singlefloats = True
-    supports_cond_call_value = True
 
     dont_keepalive_stuff = False # for tests
     with_threads = False
diff --git a/rpython/jit/backend/x86/test/test_call.py b/rpython/jit/backend/x86/test/test_call.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_call.py
@@ -0,0 +1,7 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test import test_call
+
+class TestCall(Jit386Mixin, test_call.CallTest):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_call.py
+    pass
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -7,10 +7,9 @@
 from rpython.jit.codewriter.jitcode import JitCode
 from rpython.jit.codewriter.effectinfo import (VirtualizableAnalyzer,
     QuasiImmutAnalyzer, RandomEffectsAnalyzer, effectinfo_from_writeanalyze,
-    EffectInfo, CallInfoCollection, CallShortcut)
+    EffectInfo, CallInfoCollection)
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lltype import getfunctionptr
-from rpython.flowspace.model import Constant, Variable
 from rpython.rlib import rposix
 from rpython.translator.backendopt.canraise import RaiseAnalyzer
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -215,7 +214,6 @@
         elidable = False
         loopinvariant = False
         call_release_gil_target = EffectInfo._NO_CALL_RELEASE_GIL_TARGET
-        call_shortcut = None
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
@@ -230,12 +228,6 @@
                 tgt_func, tgt_saveerr = func._call_aroundstate_target_
                 tgt_func = llmemory.cast_ptr_to_adr(tgt_func)
                 call_release_gil_target = (tgt_func, tgt_saveerr)
-            if hasattr(funcobj, 'graph'):
-                call_shortcut = self.find_call_shortcut(funcobj.graph)
-            if getattr(func, "_call_shortcut_", False):
-                assert call_shortcut is not None, (
-                    "%r: marked as @jit.call_shortcut but shortcut not found"
-                    % (func,))
         elif op.opname == 'indirect_call':
             # check that we're not trying to call indirectly some
             # function with the special flags
@@ -250,8 +242,6 @@
                     error = '@jit.loop_invariant'
                 if hasattr(graph.func, '_call_aroundstate_target_'):
                     error = '_call_aroundstate_target_'
-                if hasattr(graph.func, '_call_shortcut_'):
-                    error = '@jit.call_shortcut'
                 if not error:
                     continue
                 raise Exception(
@@ -308,7 +298,6 @@
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
             extraeffect, oopspecindex, can_invalidate, call_release_gil_target,
             extradescr, self.collect_analyzer.analyze(op, self.seen_gc),
-            call_shortcut,
         )
         #
         assert effectinfo is not None
@@ -379,76 +368,3 @@
                 if GTYPE_fieldname in jd.greenfield_info.green_fields:
                     return True
         return False
-
-    def find_call_shortcut(self, graph):
-        """Identifies graphs that start like this:
-
-           def graph(x, y, z):         def graph(x, y, z):
-               if y.field:                 r = y.field
-                   return y.field          if r: return r
-        """
-        block = graph.startblock
-        operations = block.operations
-        c_fieldname = None
-        if not operations:
-            v_inst = v_result = block.exitswitch
-        else:
-            op = operations[0]
-            if len(op.args) == 0:
-                return
-            if op.opname != 'getfield':  # check for this form:
-                v_inst = op.args[0]      #     if y is not None;
-                v_result = v_inst        #          return y
-            else:
-                operations = operations[1:]
-                [v_inst, c_fieldname] = op.args
-                v_result = op.result
-        if not isinstance(v_inst, Variable):
-            return
-        if v_result.concretetype != graph.getreturnvar().concretetype:
-            return
-        if v_result.concretetype == lltype.Void:
-            return
-        argnum = i = 0
-        while block.inputargs[i] is not v_inst:
-            if block.inputargs[i].concretetype != lltype.Void:
-                argnum += 1
-            i += 1
-        PSTRUCT = v_inst.concretetype
-        v_check = v_result
-        fastcase = True
-        for op in operations:
-            if (op.opname in ('int_is_true', 'ptr_nonzero', 'same_as')
-                    and v_check is op.args[0]):
-                v_check = op.result
-            elif op.opname == 'ptr_iszero' and v_check is op.args[0]:
-                v_check = op.result
-                fastcase = not fastcase
-            elif (op.opname in ('int_eq', 'int_ne')
-                    and v_check is op.args[0]
-                    and isinstance(op.args[1], Constant)
-                    and op.args[1].value == 0):
-                v_check = op.result
-                if op.opname == 'int_eq':
-                    fastcase = not fastcase
-            else:
-                return
-        if v_check.concretetype is not lltype.Bool:
-            return
-        if block.exitswitch is not v_check:
-            return
-
-        links = [link for link in block.exits if link.exitcase == fastcase]
-        if len(links) != 1:
-            return
-        [link] = links
-        if link.args != [v_result]:
-            return
-        if not link.target.is_final_block():
-            return
-
-        if c_fieldname is not None:
-            fielddescr = self.cpu.fielddescrof(PSTRUCT.TO, c_fieldname.value)
-        else:
-            fielddescr = None
-        return CallShortcut(argnum, fielddescr)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -117,8 +117,7 @@
                 can_invalidate=False,
                 call_release_gil_target=_NO_CALL_RELEASE_GIL_TARGET,
                 extradescrs=None,
-                can_collect=True,
-                call_shortcut=None):
+                can_collect=True):
         readonly_descrs_fields = frozenset_or_none(readonly_descrs_fields)
         readonly_descrs_arrays = frozenset_or_none(readonly_descrs_arrays)
         readonly_descrs_interiorfields = frozenset_or_none(
@@ -136,8 +135,7 @@
                extraeffect,
                oopspecindex,
                can_invalidate,
-               can_collect,
-               call_shortcut)
+               can_collect)
         tgt_func, tgt_saveerr = call_release_gil_target
         if tgt_func:
             key += (object(),)    # don't care about caching in this case
@@ -192,7 +190,6 @@
         result.oopspecindex = oopspecindex
         result.extradescrs = extradescrs
         result.call_release_gil_target = call_release_gil_target
-        result.call_shortcut = call_shortcut
         if result.check_can_raise(ignore_memoryerror=True):
             assert oopspecindex in cls._OS_CANRAISE
 
@@ -278,8 +275,7 @@
                                  call_release_gil_target=
                                      EffectInfo._NO_CALL_RELEASE_GIL_TARGET,
                                  extradescr=None,
-                                 can_collect=True,
-                                 call_shortcut=None):
+                                 can_collect=True):
     from rpython.translator.backendopt.writeanalyze import top_set
     if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
         readonly_descrs_fields = None
@@ -368,8 +364,7 @@
                       can_invalidate,
                       call_release_gil_target,
                       extradescr,
-                      can_collect,
-                      call_shortcut)
+                      can_collect)
 
 def consider_struct(TYPE, fieldname):
     if fieldType(TYPE, fieldname) is lltype.Void:
@@ -392,24 +387,6 @@
 
 # ____________________________________________________________
 
-
-class CallShortcut(object):
-    def __init__(self, argnum, fielddescr):
-        self.argnum = argnum
-        self.fielddescr = fielddescr
-
-    def __eq__(self, other):
-        return (isinstance(other, CallShortcut) and
-                self.argnum == other.argnum and
-                self.fielddescr == other.fielddescr)
-    def __ne__(self, other):
-        return not (self == other)
-    def __hash__(self):
-        return hash((self.argnum, self.fielddescr))
-
-# ____________________________________________________________
-
-
 class VirtualizableAnalyzer(BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         return op.opname in ('jit_force_virtualizable',
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -789,12 +789,20 @@
                                                 arrayfielddescr,
                                                 arraydescr)
             return []
+        # check for the string or unicode hash field
+        STRUCT = v_inst.concretetype.TO
+        if STRUCT == rstr.STR:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('strhash', [v_inst], op.result)
+        elif STRUCT == rstr.UNICODE:
+            assert c_fieldname.value == 'hash'
+            return SpaceOperation('unicodehash', [v_inst], op.result)
         # check for _immutable_fields_ hints
-        immut = v_inst.concretetype.TO._immutable_field(c_fieldname.value)
+        immut = STRUCT._immutable_field(c_fieldname.value)
         need_live = False
         if immut:
             if (self.callcontrol is not None and
-                self.callcontrol.could_be_green_field(v_inst.concretetype.TO,
+                self.callcontrol.could_be_green_field(STRUCT,
                                                       c_fieldname.value)):
                 pure = '_greenfield'
                 need_live = True
@@ -802,10 +810,9 @@
                 pure = '_pure'
         else:
             pure = ''
-        self.check_field_access(v_inst.concretetype.TO)
-        argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc')
-        descr = self.cpu.fielddescrof(v_inst.concretetype.TO,
-                                      c_fieldname.value)
+        self.check_field_access(STRUCT)
+        argname = getattr(STRUCT, '_gckind', 'gc')
+        descr = self.cpu.fielddescrof(STRUCT, c_fieldname.value)
         kind = getkind(RESULT)[0]
         if argname != 'gc':
             assert argname == 'raw'
@@ -822,7 +829,7 @@
         if immut in (IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY):
             op1.opname += "_pure"
             descr1 = self.cpu.fielddescrof(
-                v_inst.concretetype.TO,
+                STRUCT,
                 quasiimmut.get_mutate_field_name(c_fieldname.value))
             return [SpaceOperation('-live-', [], None),
                    SpaceOperation('record_quasiimmut_field',
@@ -1569,7 +1576,7 @@
             return []
         return getattr(self, 'handle_jit_marker__%s' % key)(op, jitdriver)
 
-    def rewrite_op_jit_conditional_call(self, op):
+    def _rewrite_op_cond_call(self, op, rewritten_opname):
         have_floats = False
         for arg in op.args:
             if getkind(arg.concretetype) == 'float':
@@ -1580,13 +1587,18 @@
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
         calldescr = self.callcontrol.getcalldescr(callop)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
-        op1 = self.rewrite_call(op, 'conditional_call',
+        op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
                                 calldescr=calldescr, force_ir=True)
         if self.callcontrol.calldescr_canraise(calldescr):
             op1 = [op1, SpaceOperation('-live-', [], None)]
         return op1
 
+    def rewrite_op_jit_conditional_call(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call')
+    def rewrite_op_jit_conditional_call_value(self, op):
+        return self._rewrite_op_cond_call(op, 'conditional_call_value')
+
     def handle_jit_marker__jit_merge_point(self, op, jitdriver):
         assert self.portal_jd is not None, (
             "'jit_merge_point' in non-portal graph!")
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -6,7 +6,7 @@
 from rpython.rlib import jit
 from rpython.jit.codewriter import support, call
 from rpython.jit.codewriter.call import CallControl

From pypy.commits at gmail.com  Mon Nov 28 05:25:06 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 02:25:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge py3.5-ssl
Message-ID: <583c0602.92ae1c0a.30512.6c12@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r88696:c28a4f6a70d3
Date: 2016-11-28 11:24 +0100
http://bitbucket.org/pypy/pypy/changeset/c28a4f6a70d3/

Log:	merge py3.5-ssl

diff too long, truncating to 2000 out of 15232 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,6 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
+.git/
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -145,6 +145,7 @@
 from socket import SOL_SOCKET, SO_TYPE
 import base64        # for DER-to-PEM translation
 import errno
+import warnings
 
 
 socket_error = OSError  # keep that public name in module namespace
@@ -405,12 +406,16 @@
 
     def _load_windows_store_certs(self, storename, purpose):
         certs = bytearray()
-        for cert, encoding, trust in enum_certificates(storename):
-            # CA certs are never PKCS#7 encoded
-            if encoding == "x509_asn":
-                if trust is True or purpose.oid in trust:
-                    certs.extend(cert)
-        self.load_verify_locations(cadata=certs)
+        try:
+            for cert, encoding, trust in enum_certificates(storename):
+                # CA certs are never PKCS#7 encoded
+                if encoding == "x509_asn":
+                    if trust is True or purpose.oid in trust:
+                        certs.extend(cert)
+        except PermissionError:
+            warnings.warn("unable to enumerate Windows certificate store")
+        if certs:
+            self.load_verify_locations(cadata=certs)
         return certs
 
     def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
@@ -560,7 +565,7 @@
         server hostame is set."""
         return self._sslobj.server_hostname
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to 'len' bytes from the SSL object and return them.
 
         If 'buffer' is provided, read into this buffer and return the number of
@@ -569,7 +574,7 @@
         if buffer is not None:
             v = self._sslobj.read(len, buffer)
         else:
-            v = self._sslobj.read(len or 1024)
+            v = self._sslobj.read(len)
         return v
 
     def write(self, data):
@@ -745,7 +750,8 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-            except:
+
+            except (OSError, ValueError):
                 self.close()
                 raise
 
@@ -774,7 +780,7 @@
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
 
diff --git a/lib-python/3/test/capath/0e4015b9.0 b/lib-python/3/test/capath/0e4015b9.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/0e4015b9.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/capath/ce7b8643.0 b/lib-python/3/test/capath/ce7b8643.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/ce7b8643.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/selfsigned_pythontestdotnet.pem b/lib-python/3/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/3/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/3/test/selfsigned_pythontestdotnet.pem
@@ -1,5 +1,5 @@
 -----BEGIN CERTIFICATE-----
-MIIChzCCAfCgAwIBAgIJAKGU95wKR8pSMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
 IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
 bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
@@ -8,9 +8,9 @@
 aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
 Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
 Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjKTAnMCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MA0GCSqGSIb3DQEBBQUAA4GBAIOXmdtM
-eG9qzP9TiXW/Gc/zI4cBfdCpC+Y4gOfC9bQUC7hefix4iO3+iZjgy3X/FaRxUUoV
-HKiXcXIaWqTSUWp45cSh0MbwZXudp6JIAptzdAhvvCrPKeC9i9GvxsPD4LtDAL97
-vSaxQBezA7hdxZd90/EeyMgVZgAnTCnvAWX9
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
 -----END CERTIFICATE-----
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -55,11 +55,12 @@
 SIGNED_CERTFILE2 = data_file("keycert4.pem")
 SIGNING_CA = data_file("pycacert.pem")
 
-SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem")
+REMOTE_HOST = "self-signed.pythontest.net"
+REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem")
 
 EMPTYCERT = data_file("nullcert.pem")
 BADCERT = data_file("badcert.pem")
-WRONGCERT = data_file("XXXnonexisting.pem")
+NONEXISTINGCERT = data_file("XXXnonexisting.pem")
 BADKEY = data_file("badkey.pem")
 NOKIACERT = data_file("nokia.pem")
 NULLBYTECERT = data_file("nullbytecert.pem")
@@ -276,7 +277,7 @@
         self.assertEqual(p['subjectAltName'], san)
 
     def test_DER_to_PEM(self):
-        with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f:
+        with open(CAFILE_CACERT, 'r') as f:
             pem = f.read()
         d1 = ssl.PEM_cert_to_DER_cert(pem)
         p2 = ssl.DER_cert_to_PEM_cert(d1)
@@ -366,17 +367,42 @@
                                     s.connect, (HOST, 8080))
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT)
+                ssl.wrap_socket(sock, certfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=CERTFILE, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def bad_cert_test(self, certfile):
+        """Check that trying to use the given client certificate fails"""
+        certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                   certfile)
+        sock = socket.socket()
+        self.addCleanup(sock.close)
+        with self.assertRaises(ssl.SSLError):
+            ssl.wrap_socket(sock,
+                            certfile=certfile,
+                            ssl_version=ssl.PROTOCOL_TLSv1)
+
+    def test_empty_cert(self):
+        """Wrapping with an empty cert file"""
+        self.bad_cert_test("nullcert.pem")
+
+    def test_malformed_cert(self):
+        """Wrapping with a badly formatted certificate (syntax error)"""
+        self.bad_cert_test("badcert.pem")
+
+    def test_malformed_key(self):
+        """Wrapping with a badly formatted key (syntax error)"""
+        self.bad_cert_test("badkey.pem")
+
     def test_match_hostname(self):
         def ok(cert, hostname):
             ssl.match_hostname(cert, hostname)
@@ -795,7 +821,8 @@
             self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
                              ctx.options)
             ctx.options = 0
-            self.assertEqual(0, ctx.options)
+            # Ubuntu has OP_NO_SSLv3 forced on by default
+            self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3)
         else:
             with self.assertRaises(ValueError):
                 ctx.options = 0
@@ -842,7 +869,7 @@
         ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE)
         self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE)
         with self.assertRaises(OSError) as cm:
-            ctx.load_cert_chain(WRONGCERT)
+            ctx.load_cert_chain(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_cert_chain(BADCERT)
@@ -862,7 +889,7 @@
         # Mismatching key and cert
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"):
-            ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY)
+            ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY)
         # Password protected key and cert
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD)
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode())
@@ -927,7 +954,7 @@
         self.assertRaises(TypeError, ctx.load_verify_locations)
         self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None)
         with self.assertRaises(OSError) as cm:
-            ctx.load_verify_locations(WRONGCERT)
+            ctx.load_verify_locations(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_verify_locations(BADCERT)
@@ -1003,7 +1030,7 @@
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
         with self.assertRaises(FileNotFoundError) as cm:
-            ctx.load_dh_params(WRONGCERT)
+            ctx.load_dh_params(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(ssl.SSLError) as cm:
             ctx.load_dh_params(CERTFILE)
@@ -1080,7 +1107,7 @@
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 0, 'crl': 0, 'x509': 1})
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 1, 'crl': 0, 'x509': 2})
 
@@ -1090,8 +1117,8 @@
         # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.get_ca_certs(), [])
-        # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        # but CAFILE_CACERT is a CA cert
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.get_ca_certs(),
             [{'issuer': ((('organizationName', 'Root CA'),),
                          (('organizationalUnitName', 'http://www.cacert.org'),),
@@ -1107,7 +1134,7 @@
                           (('emailAddress', 'support at cacert.org'),)),
               'version': 3}])
 
-        with open(SVN_PYTHON_ORG_ROOT_CERT) as f:
+        with open(CAFILE_CACERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
         self.assertEqual(ctx.get_ca_certs(True), [der])
@@ -1345,11 +1372,11 @@
 class NetworkedTests(unittest.TestCase):
 
     def test_connect(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
@@ -1358,27 +1385,27 @@
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED)
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                   s.connect, ("svn.python.org", 443))
+                                   s.connect, (REMOTE_HOST, 443))
             s.close()
 
             # this should succeed because we specify the root cert
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
 
     def test_connect_ex(self):
         # Issue #11326: check connect_ex() implementation
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                self.assertEqual(0, s.connect_ex(("svn.python.org", 443)))
+                self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443)))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
@@ -1386,14 +1413,14 @@
     def test_non_blocking_connect_ex(self):
         # Issue #11326: non-blocking connect_ex() should allow handshake
         # to proceed after the socket gets ready.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.setblocking(False)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
                 self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
                 # Wait for connect to finish
@@ -1415,58 +1442,62 @@
     def test_timeout_connect_ex(self):
         # Issue #12065: on a timeout, connect_ex() should return the original
         # errno (mimicking the behaviour of non-SSL sockets).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.settimeout(0.0000001)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 if rc == 0:
-                    self.skipTest("svn.python.org responded too quickly")
+                    self.skipTest("REMOTE_HOST responded too quickly")
                 self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK))
             finally:
                 s.close()
 
     def test_connect_ex_error(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                rc = s.connect_ex(("svn.python.org", 444))
+                rc = s.connect_ex((REMOTE_HOST, 444))
                 # Issue #19919: Windows machines or VMs hosted on Windows
                 # machines sometimes return EWOULDBLOCK.
-                self.assertIn(rc, (errno.ECONNREFUSED, errno.EWOULDBLOCK))
+                errors = (
+                    errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT,
+                    errno.EWOULDBLOCK,
+                )
+                self.assertIn(rc, errors)
             finally:
                 s.close()
 
     def test_connect_with_context(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             # Same as test_connect, but with a separately created context
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
             # Same with a server hostname
             s = ctx.wrap_socket(socket.socket(socket.AF_INET),
-                                server_hostname="svn.python.org")
-            s.connect(("svn.python.org", 443))
+                                server_hostname=REMOTE_HOST)
+            s.connect((REMOTE_HOST, 443))
             s.close()
             # This should fail because we have no verification certs
             ctx.verify_mode = ssl.CERT_REQUIRED
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                    s.connect, ("svn.python.org", 443))
+                                    s.connect, (REMOTE_HOST, 443))
             s.close()
             # This should succeed because we specify the root cert
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1479,12 +1510,12 @@
         # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must
         # contain both versions of each certificate (same content, different
         # filename) for this test to be portable across OpenSSL releases.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1495,7 +1526,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=BYTES_CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1503,15 +1534,15 @@
                 s.close()
 
     def test_connect_cadata(self):
-        with open(CAFILE_CACERT) as f:
+        with open(REMOTE_ROOT_CERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=pem)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1520,7 +1551,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=der)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1529,9 +1560,9 @@
         # Issue #5238: creating a file-like object with makefile() shouldn't
         # delay closing the underlying "real socket" (here tested with its
         # file descriptor, hence skipping the test under Windows).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
-            ss.connect(("svn.python.org", 443))
+            ss.connect((REMOTE_HOST, 443))
             fd = ss.fileno()
             f = ss.makefile()
             f.close()
@@ -1545,9 +1576,9 @@
             self.assertEqual(e.exception.errno, errno.EBADF)
 
     def test_non_blocking_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = socket.socket(socket.AF_INET)
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             s.setblocking(False)
             s = ssl.wrap_socket(s,
                                 cert_reqs=ssl.CERT_NONE,
@@ -1590,12 +1621,12 @@
                 if support.verbose:
                     sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))
 
-        _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT)
+        _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT)
         if support.IPV6_ENABLED:
             _test_get_server_certificate('ipv6.google.com', 443)
 
     def test_ciphers(self):
-        remote = ("svn.python.org", 443)
+        remote = (REMOTE_HOST, 443)
         with support.transient_internet(remote[0]):
             with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                  cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
@@ -1640,13 +1671,13 @@
 
     def test_get_ca_certs_capath(self):
         # capath certs are loaded on request
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             self.assertEqual(ctx.get_ca_certs(), [])
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1657,12 +1688,12 @@
     @needs_sni
     def test_context_setget(self):
         # Check that the context of a connected socket can be replaced.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = socket.socket(socket.AF_INET)
             with ctx1.wrap_socket(s) as ss:
-                ss.connect(("svn.python.org", 443))
+                ss.connect((REMOTE_HOST, 443))
                 self.assertIs(ss.context, ctx1)
                 self.assertIs(ss._sslobj.context, ctx1)
                 ss.context = ctx2
@@ -1683,13 +1714,8 @@
             try:
                 ret = func(*args)
             except ssl.SSLError as e:
-                # Note that we get a spurious -1/SSL_ERROR_SYSCALL for
-                # non-blocking IO. The SSL_shutdown manpage hints at this.
-                # It *should* be safe to just ignore SYS_ERROR_SYSCALL because
-                # with a Memory BIO there's no syscalls (for IO at least).
                 if e.errno not in (ssl.SSL_ERROR_WANT_READ,
-                                   ssl.SSL_ERROR_WANT_WRITE,
-                                   ssl.SSL_ERROR_SYSCALL):
+                                   ssl.SSL_ERROR_WANT_WRITE):
                     raise
                 errno = e.errno
             # Get any data from the outgoing BIO irrespective of any error, and
@@ -1712,36 +1738,46 @@
         return ret
 
     def test_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        # NOTE: this test has been modified, CPython in newer versions
+        # removed the ability to get the shared ciphers of the session, but
+        # they always return the cipher of the ssl context. This test is fully
+        # removed in later versions
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             ctx.check_hostname = True
-            sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org')
+            sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
-            self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            try:
+                self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            except ssl.SSLSyscallError:
+                # self-signed.pythontest.net probably shuts down the TCP
+                # connection without sending a secure shutdown message, and
+                # this is reported as SSL_ERROR_SYSCALL
+                pass
             self.assertRaises(ssl.SSLError, sslobj.write, b'foo')
             sock.close()
 
     def test_read_write_data(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -1838,14 +1874,7 @@
                 else:
                     self.sock.close()
 
-            # PyPy change
             def run(self):
-                try:
-                    self._run()
-                finally:
-                    self.close()
-
-            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
@@ -2091,36 +2120,6 @@
             self.active = False
             self.server.close()
 
-    def bad_cert_test(certfile):
-        """
-        Launch a server with CERT_REQUIRED, and check that trying to
-        connect to it with the given client certificate fails.
-        """
-        server = ThreadedEchoServer(CERTFILE,
-                                    certreqs=ssl.CERT_REQUIRED,
-                                    cacerts=CERTFILE, chatty=False,
-                                    connectionchatty=False)
-        with server:
-            try:
-                with socket.socket() as sock:
-                    s = ssl.wrap_socket(sock,
-                                        certfile=certfile,
-                                        ssl_version=ssl.PROTOCOL_TLSv1)
-                    s.connect((HOST, server.port))
-            except ssl.SSLError as x:
-                if support.verbose:
-                    sys.stdout.write("\nSSLError is %s\n" % x.args[1])
-            except OSError as x:
-                if support.verbose:
-                    sys.stdout.write("\nOSError is %s\n" % x.args[1])
-            except OSError as x:
-                if x.errno != errno.ENOENT:
-                    raise
-                if support.verbose:
-                    sys.stdout.write("\OSError is %s\n" % str(x))
-            else:
-                raise AssertionError("Use of invalid cert should have failed!")
-
     def server_params_test(client_context, server_context, indata=b"FOO\n",
                            chatty=True, connectionchatty=False, sni_name=None):
         """
@@ -2361,22 +2360,38 @@
                                                 "check_hostname requires server_hostname"):
                         context.wrap_socket(s)
 
-        def test_empty_cert(self):
-            """Connecting with an empty cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                      "nullcert.pem"))
-        def test_malformed_cert(self):
-            """Connecting with a badly formatted certificate (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badcert.pem"))
-        def test_nonexisting_cert(self):
-            """Connecting with a non-existing cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "wrongcert.pem"))
-        def test_malformed_key(self):
-            """Connecting with a badly formatted key (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badkey.pem"))
+        def test_wrong_cert(self):
+            """Connecting when the server rejects the client's certificate
+
+            Launch a server with CERT_REQUIRED, and check that trying to
+            connect to it with a wrong client certificate fails.
+            """
+            certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                       "wrongcert.pem")
+            server = ThreadedEchoServer(CERTFILE,
+                                        certreqs=ssl.CERT_REQUIRED,
+                                        cacerts=CERTFILE, chatty=False,
+                                        connectionchatty=False)
+            with server, \
+                    socket.socket() as sock, \
+                    ssl.wrap_socket(sock,
+                                        certfile=certfile,
+                                        ssl_version=ssl.PROTOCOL_TLSv1) as s:
+                try:
+                    # Expect either an SSL error about the server rejecting
+                    # the connection, or a low-level connection reset (which
+                    # sometimes happens on Windows)
+                    s.connect((HOST, server.port))
+                except ssl.SSLError as e:
+                    if support.verbose:
+                        sys.stdout.write("\nSSLError is %r\n" % e)
+                except OSError as e:
+                    if e.errno != errno.ECONNRESET:
+                        raise
+                    if support.verbose:
+                        sys.stdout.write("\nsocket.error is %r\n" % e)
+                else:
+                    self.fail("Use of invalid cert should have failed!")
 
         def test_rude_shutdown(self):
             """A brutal shutdown of an SSL server should raise an OSError
@@ -2782,6 +2797,20 @@
                         # consume data
                         s.read()
 
+                data = b"data"
+
+                # read(-1, buffer) is supported, even though read(-1) is not
+                s.send(data)
+                buffer = bytearray(len(data))
+                self.assertEqual(s.read(-1, buffer), len(data))
+                self.assertEqual(buffer, data)
+
+                # recv/read(0) should return no data
+                s.send(data)
+                self.assertEqual(s.recv(0), b"")
+                self.assertEqual(s.read(0), b"")
+                self.assertEqual(s.read(), data)
+
                 # Make sure sendmsg et al are disallowed to avoid
                 # inadvertent disclosure of data and/or corruption
                 # of the encrypted data stream
@@ -2791,6 +2820,10 @@
                                   s.recvmsg_into, bytearray(100))
 
                 s.write(b"over\n")
+
+                self.assertRaises(ValueError, s.recv, -1)
+                self.assertRaises(ValueError, s.read, -1)
+
                 s.close()
 
         def test_nonblocking_send(self):
@@ -3253,18 +3286,31 @@
             self.assertIn("TypeError", stderr.getvalue())
 
         def test_shared_ciphers(self):
+            # NOTE: This test case has been copied from master (~3.6)
+            # reason is the method SSL_get_ciphers (before that it used
+            # get_ciphers of the session which is not public)
             server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             server_context.load_cert_chain(SIGNED_CERTFILE)
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            client_context.set_ciphers("RC4")
-            server_context.set_ciphers("AES:RC4")
+            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
+                client_context.set_ciphers("AES128:AES256")
+                server_context.set_ciphers("AES256")
+                alg1 = "AES256"
+                alg2 = "AES-256"
+            else:
+                client_context.set_ciphers("AES:3DES")
+                server_context.set_ciphers("3DES")
+                alg1 = "3DES"
+                alg2 = "DES-CBC3"
+
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                self.assertIn("RC4", name.split("-"))
+                if not alg1 in name.split("-") and alg2 not in name:
+                    self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -3332,7 +3378,7 @@
             pass
 
     for filename in [
-        CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE,
+        CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE,
         ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY,
         SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA,
         BADCERT, BADKEY, EMPTYCERT]:
diff --git a/lib-python/3/test/wrongcert.pem b/lib-python/3/test/wrongcert.pem
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/wrongcert.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH
+FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T
+f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB
+AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq
+1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW
+7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg
+SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe
+19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg
+ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666
+lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs
+ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv
+frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk
+2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6
++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK
+24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G
+A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9
+OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt
+U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ
+fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E
+usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+
+43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw
+eSHj5jpC8iZKjCHBn+mAi4cQ514=
+-----END CERTIFICATE-----
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Vim
+
+*.swp
+*.swo
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,16 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..180c02ff82d3363f34a334aae22c9876d4c96481
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,85 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *,
+                       SecKeychainRef, CFArrayRef *);
+OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *);
+OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags,
+                       const SecItemImportExportKeyParameters *, CFDataRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecItem.h>
+"""
+
+TYPES = """
+const CFTypeRef kSecAttrKeyType;
+const CFTypeRef kSecAttrKeySizeInBits;
+const CFTypeRef kSecAttrIsPermanent;
+const CFTypeRef kSecAttrKeyTypeRSA;
+const CFTypeRef kSecAttrKeyTypeDSA;
+const CFTypeRef kSecUseKeychain;
+"""
+
+FUNCTIONS = """
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py
@@ -0,0 +1,24 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKey.h>
+"""
+
+TYPES = """
+typedef ... *SecKeyRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *);
+size_t SecKeyGetBlockSize(SecKeyRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py
@@ -0,0 +1,25 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecKeychain.h>
+"""
+
+TYPES = """
+typedef ... *SecKeychainRef;
+"""
+
+FUNCTIONS = """
+OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean,
+                           SecAccessRef, SecKeychainRef *);
+OSStatus SecKeychainDelete(SecKeychainRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecPolicy.h>
+"""
+
+TYPES = """
+typedef ... *SecPolicyRef;
+"""
+
+FUNCTIONS = """
+SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecDigestTransform.h>
+#include <Security/SecSignVerifyTransform.h>
+#include <Security/SecEncryptTransform.h>
+"""
+
+TYPES = """
+typedef ... *SecTransformRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+CFStringRef kSecEncryptionMode;
+CFStringRef kSecEncryptKey;
+CFStringRef kSecIVKey;
+CFStringRef kSecModeCBCKey;
+CFStringRef kSecModeCFBKey;
+CFStringRef kSecModeECBKey;
+CFStringRef kSecModeNoneKey;
+CFStringRef kSecModeOFBKey;
+CFStringRef kSecOAEPEncodingParametersAttributeName;
+CFStringRef kSecPaddingKey;
+CFStringRef kSecPaddingNoneKey;
+CFStringRef kSecPaddingOAEPKey;
+CFStringRef kSecPaddingPKCS1Key;
+CFStringRef kSecPaddingPKCS5Key;
+CFStringRef kSecPaddingPKCS7Key;
+
+const CFStringRef kSecTransformInputAttributeName;
+const CFStringRef kSecTransformOutputAttributeName;
+const CFStringRef kSecTransformDebugAttributeName;
+const CFStringRef kSecTransformTransformName;
+const CFStringRef kSecTransformAbortAttributeName;
+
+CFStringRef kSecInputIsAttributeName;
+CFStringRef kSecInputIsPlainText;
+CFStringRef kSecInputIsDigest;
+CFStringRef kSecInputIsRaw;
+
+const CFStringRef kSecDigestTypeAttribute;
+const CFStringRef kSecDigestLengthAttribute;
+const CFStringRef kSecDigestMD5;
+const CFStringRef kSecDigestSHA1;
+const CFStringRef kSecDigestSHA2;
+"""
+
+FUNCTIONS = """
+Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef,
+                                 CFErrorRef *);
+SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *);
+SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *);
+SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ;
+CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecTrust.h>
+"""
+
+TYPES = """
+typedef ... *SecTrustRef;
+typedef uint32_t SecTrustResultType;
+
+enum {
+    kSecTrustResultInvalid,
+    kSecTrustResultProceed,
+    kSecTrustResultDeny,
+    kSecTrustResultUnspecified,
+    kSecTrustResultRecoverableTrustFailure,
+    kSecTrustResultFatalTrustFailure,
+    kSecTrustResultOtherError
+};
+"""
+
+FUNCTIONS = """
+OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *);
+OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *);
+"""
+
+MACROS = """
+/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this
+ * has to go here for compatibility.
+ */
+OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *);
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py
@@ -0,0 +1,308 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecureTransport.h>
+"""
+
+TYPES = """
+typedef ... *SSLContextRef;
+typedef const void *SSLConnectionRef;

From pypy.commits at gmail.com  Mon Nov 28 09:15:26 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 06:15:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: update the _ssl implementations with
 review suggestions
Message-ID: <583c3bfe.44e61c0a.17aa8.c54f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88698:50ed199b9bf4
Date: 2016-11-28 15:14 +0100
http://bitbucket.org/pypy/pypy/changeset/50ed199b9bf4/

Log:	update the _ssl implementations with review suggestions

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,8 @@
         SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE,
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
+from select import poll, POLLIN, POLLOUT, select
+from enum import IntEnum as _IntEnum
 
 OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
 OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER
@@ -77,7 +79,6 @@
 
 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
 
-from enum import IntEnum as _IntEnum
 _IntEnum._convert('_SSLMethod', __name__,
         lambda name: name.startswith('PROTOCOL_'))
 
@@ -120,7 +121,6 @@
 def _Cryptography_pem_password_cb(buf, size, rwflag, userdata):
     pw_info = ffi.from_handle(userdata)
 
-    # TODO PySSL_END_ALLOW_THREADS_S(pw_info->thread_state);
     password = pw_info.password
 
     if pw_info.callable:
@@ -140,7 +140,6 @@
         pw_info.operationerror = ValueError("password cannot be longer than %d bytes" % size)
         return 0
 
-    #PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state);
     ffi.memmove(buf, password, len(password))
     return len(password)
 
@@ -150,7 +149,12 @@
 else:
     Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
 
-from select import poll, POLLIN, POLLOUT, select
+if hasattr(time, 'monotonic'):
+    def _monotonic_clock():
+        return time.monotonic()
+else:
+    def _monotonic_clock():
+        return time.clock_gettime(time.CLOCK_MONOTONIC)
 
 HAVE_POLL = True
 
@@ -177,10 +181,9 @@
     if HAVE_POLL:
         p.register(sock.fileno(), POLLOUT | POLLIN)
 
-        #PySSL_BEGIN_ALLOW_THREADS
         rc = len(p.poll(timeout * 1000.0))
-        #PySSL_END_ALLOW_THREADS
     else:
+        # currently disabled, see HAVE_POLL
         fd = sock.fileno()
         #if (!_PyIsSelectable_fd(s->sock_fd))
         #    return SOCKET_TOO_LARGE_FOR_SELECT;
@@ -217,7 +220,7 @@
         lib.ERR_clear_error()
         self.ssl = ssl = lib.SSL_new(ctx)
 
-        self._app_data_handle = ffi.new_handle(self) # TODO release later
+        self._app_data_handle = ffi.new_handle(self)
         lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
         if sock:
             lib.SSL_set_fd(ssl, sock.fileno())
@@ -247,13 +250,11 @@
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), 1)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), 1)
 
-        #PySSL_BEGIN_ALLOW_THREADS
         if socket_type == SSL_CLIENT:
             lib.SSL_set_connect_state(ssl)
         else:
             lib.SSL_set_accept_state(ssl)
         self.socket_type = socket_type
-        #PySSL_END_ALLOW_THREADS
 
         if sock:
             self.socket = weakref.ref(sock)
@@ -270,6 +271,9 @@
         self.server_hostname = None
         self.socket = None
 
+    def __del__(self):
+        self._app_data_handle = None
+
     @property
     def owner(self):
         if self._owner is None:
@@ -309,8 +313,7 @@
         has_timeout = timeout > 0
         deadline = -1
         if has_timeout:
-            # REVIEW, cpython uses a monotonic clock here
-            deadline = time.time() + timeout;
+            deadline = _monotonic_clock() + timeout;
         # Actually negotiate SSL connection
         # XXX If SSL_do_handshake() returns 0, it's also a failure.
         while True:
@@ -323,7 +326,7 @@
 
             if has_timeout:
                 # REIVIEW monotonic clock?
-                timeout = deadline - time.time()
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -347,7 +350,6 @@
 
         if self.peer_cert != ffi.NULL:
             lib.X509_free(self.peer_cert)
-        #PySSL_BEGIN_ALLOW_THREADS
         self.peer_cert = lib.SSL_get_peer_certificate(ssl)
         #PySSL_END_ALLOW_THREADS
         self.handshake_done = 1
@@ -387,8 +389,7 @@
 
         has_timeout = timeout > 0
         if has_timeout:
-            # TODO monotonic clock?
-            deadline = time.time() + timeout
+            deadline = _monotonic_clock() + timeout
 
         sockstate = _ssl_select(sock, 1, timeout)
         if sockstate == SOCKET_HAS_TIMED_OUT:
@@ -399,16 +400,13 @@
             raise ssl_error("Underlying socket too large for select().")
 
         while True:
-            #PySSL_START_ALLOW_THREADS
             length = lib.SSL_write(self.ssl, b, len(b))
             err = lib.SSL_get_error(self.ssl, length)
-            #PySSL_END_ALLOW_THREADS
 
             check_signals()
 
             if has_timeout:
-                # TODO monotonic clock
-                timeout = deadline - time.time()
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -459,20 +457,17 @@
         timeout = _socket_timeout(sock)
         has_timeout = timeout > 0
         if has_timeout:
-            # TODO monotonic clock?
-            deadline = time.time() + timeout
+            deadline = _monotonic_clock() + timeout
 
         shutdown = False
         while True:
-            #PySSL_BEGIN_ALLOW_THREADS
             count = lib.SSL_read(self.ssl, mem, length);
             err = lib.SSL_get_error(self.ssl, count);
-            #PySSL_END_ALLOW_THREADS
 
             check_signals()
 
             if has_timeout:
-                timeout = deadline - time.time() # TODO ? _PyTime_GetMonotonicClock();
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -591,13 +586,11 @@
 
         has_timeout = (timeout > 0);
         if has_timeout:
-            # TODO monotonic clock
-            deadline = time.time() + timeout;
+            deadline = _monotonic_clock() + timeout;
 
         zeros = 0
 
         while True:
-            # TODO PySSL_BEGIN_ALLOW_THREADS
             # Disable read-ahead so that unwrap can work correctly.
             # Otherwise OpenSSL might read in too much data,
             # eating clear text data that happens to be
@@ -609,7 +602,6 @@
             if self.shutdown_seen_zero:
                 lib.SSL_set_read_ahead(self.ssl, 0)
             err = lib.SSL_shutdown(self.ssl)
-            # TODO PySSL_END_ALLOW_THREADS
 
             # If err == 1, a secure shutdown with SSL_shutdown() is complete
             if err > 0:
@@ -626,8 +618,7 @@
                 continue
 
             if has_timeout:
-                # TODO monotonic clock
-                timeout = deadline - time.time() #_PyTime_GetMonotonicClock();
+                timeout = deadline - _monotonic_clock()
 
             # Possibly retry shutdown until timeout or failure
             ssl_err = lib.SSL_get_error(self.ssl, err)
@@ -657,9 +648,7 @@
             return None
 
     def pending(self):
-        # TODO PySSL_BEGIN_ALLOW_THREADS
         count = lib.SSL_pending(self.ssl)
-        # TODO PySSL_END_ALLOW_THREADS
         if count < 0:
             raise pyssl_error(self, count)
         else:
@@ -692,8 +681,7 @@
 
 
 def _fs_decode(name):
-    # TODO return PyUnicode_DecodeFSDefault(short_name);
-    return name.decode('utf-8')
+    return name.decode(sys.getfilesystemencoding())
 
 
 def cipher_to_tuple(cipher):
@@ -747,7 +735,7 @@
         else:
             raise ValueError("invalid protocol version")
 
-        self.ctx = lib.SSL_CTX_new(method)
+        self.ctx = ffi.gc(lib.SSL_CTX_new(method), lib.SSL_CTX_free)
         if self.ctx == ffi.NULL: 
             raise ssl_error("failed to allocate SSL context")
 
@@ -1049,15 +1037,6 @@
         return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
 
 
-    def __del__(self):
-        # REVIEW, is this done properly? In RPython the a finalizer
-        # function is added
-        ctx = self.ctx
-        if ctx:
-            self.ctx = None
-            lib.SSL_CTX_free(ctx)
-
-
     def session_stats(self):
         stats = {}
         for name, ssl_func in SSL_CTX_STATS:
@@ -1273,7 +1252,6 @@
         self.ctx = ctx
 
 SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
-TEST = None
 
 def _asn1obj2py(obj):
     nid = lib.OBJ_obj2nid(obj)
@@ -1376,7 +1354,6 @@
 
 
 RAND_status = lib.RAND_status
-RAND_add = lib.RAND_add
 
 def _RAND_bytes(count, pseudo):
     if count < 0:
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -278,7 +278,6 @@
         lib.BIO_free(cert)
         raise ssl_error("Can't malloc memory to read file")
 
-    # REVIEW how to encode this properly?
     epath = path.encode()
     if lib.BIO_read_filename(cert, epath) <= 0:
         lib.BIO_free(cert)
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -14,38 +14,21 @@
     return ffi.buffer(char_ptr, length)[:]
 
 def _str_to_ffi_buffer(view, zeroterm=False):
-    # REVIEW unsure how to solve this. might be easy:
-    # str does not support buffer protocol.
-    # I think a user should really encode the string before it is 
-    # passed here!
-    if isinstance(view, str):
-        enc = view.encode()
-        if zeroterm:
-            return ffi.from_buffer(enc + b'\x00')
-        else:
-            return ffi.from_buffer(enc)
-    else:
-        if isinstance(view, memoryview):
-            # TODO pypy limitation StringBuffer does not allow
-            # to get a raw address to the string!
-            view = bytes(view)
-        if zeroterm:
-            return ffi.from_buffer(view + b'\x00')
-        else:
-            return ffi.from_buffer(view)
+    if zeroterm:
+        # only two cases use zeroterm=True, those are rather 'short' strings
+        return ffi.from_buffer(bytes(view)+b'\x00')
+
+    try:
+        buf = ffi.from_buffer(view)
+        return buf
+    except TypeError:
+        return ffi.from_buffer(bytes(view))
 
 def _str_from_buf(buf):
     return ffi.string(buf).decode('utf-8')
 
 def _cstr_decode_fs(buf):
-#define CONVERT(info, target) { \
-#        const char *tmp = (info); \
-#        target = NULL; \
-#        if (!tmp) { Py_INCREF(Py_None); target = Py_None; } \
-#        else if ((target = PyUnicode_DecodeFSDefault(tmp)) == NULL) { \
-#            target = PyBytes_FromString(tmp); } \
-#        if (!target) goto error; \
-#    }
-    # REVIEW
+    if buf == ffi.NULL:
+        return None
     return ffi.string(buf).decode(sys.getfilesystemencoding())
 
diff --git a/lib_pypy/ssl.py b/lib_pypy/ssl.py
deleted file mode 100644
--- a/lib_pypy/ssl.py
+++ /dev/null
@@ -1,1148 +0,0 @@
-# Wrapper module for _ssl, providing some additional facilities
-# implemented in Python.  Written by Bill Janssen.
-
-"""This module provides some more Pythonic support for SSL.
-
-Object types:
-
-  SSLSocket -- subtype of socket.socket which does SSL over the socket
-
-Exceptions:
-
-  SSLError -- exception raised for I/O errors
-
-Functions:
-
-  cert_time_to_seconds -- convert time string used for certificate
-                          notBefore and notAfter functions to integer
-                          seconds past the Epoch (the time values
-                          returned from time.time())
-
-  fetch_server_certificate (HOST, PORT) -- fetch the certificate provided
-                          by the server running on HOST at port PORT.  No
-                          validation of the certificate is performed.
-
-Integer constants:
-
-SSL_ERROR_ZERO_RETURN
-SSL_ERROR_WANT_READ
-SSL_ERROR_WANT_WRITE
-SSL_ERROR_WANT_X509_LOOKUP
-SSL_ERROR_SYSCALL
-SSL_ERROR_SSL
-SSL_ERROR_WANT_CONNECT
-
-SSL_ERROR_EOF
-SSL_ERROR_INVALID_ERROR_CODE
-
-The following group define certificate requirements that one side is
-allowing/requiring from the other side:
-
-CERT_NONE - no certificates from the other side are required (or will
-            be looked at if provided)
-CERT_OPTIONAL - certificates are not required, but if provided will be
-                validated, and if validation fails, the connection will
-                also fail
-CERT_REQUIRED - certificates are required, and will be validated, and
-                if validation fails, the connection will also fail
-
-The following constants identify various SSL protocol variants:
-
-PROTOCOL_SSLv2
-PROTOCOL_SSLv3
-PROTOCOL_SSLv23
-PROTOCOL_TLSv1
-PROTOCOL_TLSv1_1
-PROTOCOL_TLSv1_2
-
-The following constants identify various SSL alert message descriptions as per
-http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6
-
-ALERT_DESCRIPTION_CLOSE_NOTIFY
-ALERT_DESCRIPTION_UNEXPECTED_MESSAGE
-ALERT_DESCRIPTION_BAD_RECORD_MAC
-ALERT_DESCRIPTION_RECORD_OVERFLOW
-ALERT_DESCRIPTION_DECOMPRESSION_FAILURE
-ALERT_DESCRIPTION_HANDSHAKE_FAILURE
-ALERT_DESCRIPTION_BAD_CERTIFICATE
-ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE
-ALERT_DESCRIPTION_CERTIFICATE_REVOKED
-ALERT_DESCRIPTION_CERTIFICATE_EXPIRED
-ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN
-ALERT_DESCRIPTION_ILLEGAL_PARAMETER
-ALERT_DESCRIPTION_UNKNOWN_CA
-ALERT_DESCRIPTION_ACCESS_DENIED
-ALERT_DESCRIPTION_DECODE_ERROR
-ALERT_DESCRIPTION_DECRYPT_ERROR
-ALERT_DESCRIPTION_PROTOCOL_VERSION
-ALERT_DESCRIPTION_INSUFFICIENT_SECURITY
-ALERT_DESCRIPTION_INTERNAL_ERROR
-ALERT_DESCRIPTION_USER_CANCELLED
-ALERT_DESCRIPTION_NO_RENEGOTIATION
-ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION
-ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
-ALERT_DESCRIPTION_UNRECOGNIZED_NAME
-ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE
-ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE
-ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY
-"""
-
-import ipaddress
-import textwrap
-import re
-import sys
-import os
-from collections import namedtuple
-from enum import Enum as _Enum, IntEnum as _IntEnum
-
-import _ssl             # if we can't import it, let the error propagate
-
-from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
-from _ssl import _SSLContext, MemoryBIO
-from _ssl import (
-    SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError,
-    SSLSyscallError, SSLEOFError,
-    )
-from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj
-from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes
-try:
-    from _ssl import RAND_egd
-except ImportError:
-    # LibreSSL does not provide RAND_egd
-    pass
-
-def _import_symbols(prefix):
-    for n in dir(_ssl):
-        if n.startswith(prefix):
-            globals()[n] = getattr(_ssl, n)
-
-_import_symbols('OP_')
-_import_symbols('ALERT_DESCRIPTION_')
-_import_symbols('SSL_ERROR_')
-_import_symbols('VERIFY_')
-
-from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN
-
-from _ssl import _OPENSSL_API_VERSION
-
-_IntEnum._convert(
-        '_SSLMethod', __name__,
-        lambda name: name.startswith('PROTOCOL_'),
-        source=_ssl)
-
-_PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()}
-
-try:
-    _SSLv2_IF_EXISTS = PROTOCOL_SSLv2
-except NameError:
-    _SSLv2_IF_EXISTS = None
-
-if sys.platform == "win32":
-    from _ssl import enum_certificates, enum_crls
-
-from socket import socket, AF_INET, SOCK_STREAM, create_connection
-from socket import SOL_SOCKET, SO_TYPE
-import base64        # for DER-to-PEM translation
-import errno
-import warnings
-
-
-socket_error = OSError  # keep that public name in module namespace
-
-if _ssl.HAS_TLS_UNIQUE:
-    CHANNEL_BINDING_TYPES = ['tls-unique']
-else:
-    CHANNEL_BINDING_TYPES = []
-
-# Disable weak or insecure ciphers by default
-# (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL')
-# Enable a better set of ciphers by default
-# This list has been explicitly chosen to:
-#   * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
-#   * Prefer ECDHE over DHE for better performance
-#   * Prefer any AES-GCM over any AES-CBC for better performance and security
-#   * Then Use HIGH cipher suites as a fallback
-#   * Then Use 3DES as fallback which is secure but slow
-#   * Disable NULL authentication, NULL encryption, and MD5 MACs for security
-#     reasons
-_DEFAULT_CIPHERS = (
-    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
-    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
-    '!eNULL:!MD5'
-)
-
-# Restricted and more secure ciphers for the server side
-# This list has been explicitly chosen to:
-#   * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
-#   * Prefer ECDHE over DHE for better performance
-#   * Prefer any AES-GCM over any AES-CBC for better performance and security
-#   * Then Use HIGH cipher suites as a fallback
-#   * Then Use 3DES as fallback which is secure but slow
-#   * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, and RC4 for
-#     security reasons
-_RESTRICTED_SERVER_CIPHERS = (
-    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
-    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
-    '!eNULL:!MD5:!DSS:!RC4'
-)
-
-
-class CertificateError(ValueError):
-    pass
-
-
-def _dnsname_match(dn, hostname, max_wildcards=1):
-    """Matching according to RFC 6125, section 6.4.3
-
-    http://tools.ietf.org/html/rfc6125#section-6.4.3
-    """
-    pats = []
-    if not dn:
-        return False
-
-    leftmost, *remainder = dn.split(r'.')
-
-    wildcards = leftmost.count('*')
-    if wildcards > max_wildcards:
-        # Issue #17980: avoid denials of service by refusing more
-        # than one wildcard per fragment.  A survey of established
-        # policy among SSL implementations showed it to be a
-        # reasonable choice.
-        raise CertificateError(
-            "too many wildcards in certificate DNS name: " + repr(dn))
-
-    # speed up common case w/o wildcards
-    if not wildcards:
-        return dn.lower() == hostname.lower()
-
-    # RFC 6125, section 6.4.3, subitem 1.
-    # The client SHOULD NOT attempt to match a presented identifier in which
-    # the wildcard character comprises a label other than the left-most label.
-    if leftmost == '*':
-        # When '*' is a fragment by itself, it matches a non-empty dotless
-        # fragment.
-        pats.append('[^.]+')
-    elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
-        # RFC 6125, section 6.4.3, subitem 3.
-        # The client SHOULD NOT attempt to match a presented identifier
-        # where the wildcard character is embedded within an A-label or
-        # U-label of an internationalized domain name.
-        pats.append(re.escape(leftmost))
-    else:
-        # Otherwise, '*' matches any dotless string, e.g. www*
-        pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
-
-    # add the remaining fragments, ignore any wildcards
-    for frag in remainder:
-        pats.append(re.escape(frag))
-
-    pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
-    return pat.match(hostname)
-
-
-def _ipaddress_match(ipname, host_ip):
-    """Exact matching of IP addresses.
-
-    RFC 6125 explicitly doesn't define an algorithm for this
-    (section 1.7.2 - "Out of Scope").
-    """
-    # OpenSSL may add a trailing newline to a subjectAltName's IP address
-    ip = ipaddress.ip_address(ipname.rstrip())
-    return ip == host_ip
-
-
-def match_hostname(cert, hostname):
-    """Verify that *cert* (in decoded format as returned by
-    SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
-    rules are followed, but IP addresses are not accepted for *hostname*.
-
-    CertificateError is raised on failure. On success, the function
-    returns nothing.
-    """
-    if not cert:
-        raise ValueError("empty or no certificate, match_hostname needs a "
-                         "SSL socket or SSL context with either "
-                         "CERT_OPTIONAL or CERT_REQUIRED")
-    try:
-        host_ip = ipaddress.ip_address(hostname)
-    except ValueError:
-        # Not an IP address (common case)
-        host_ip = None
-    dnsnames = []
-    san = cert.get('subjectAltName', ())
-    for key, value in san:
-        if key == 'DNS':
-            if host_ip is None and _dnsname_match(value, hostname):
-                return
-            dnsnames.append(value)
-        elif key == 'IP Address':
-            if host_ip is not None and _ipaddress_match(value, host_ip):
-                return
-            dnsnames.append(value)
-    if not dnsnames:
-        # The subject is only checked when there is no dNSName entry
-        # in subjectAltName
-        for sub in cert.get('subject', ()):
-            for key, value in sub:
-                # XXX according to RFC 2818, the most specific Common Name
-                # must be used.
-                if key == 'commonName':
-                    if _dnsname_match(value, hostname):
-                        return
-                    dnsnames.append(value)
-    if len(dnsnames) > 1:
-        raise CertificateError("hostname %r "
-            "doesn't match either of %s"
-            % (hostname, ', '.join(map(repr, dnsnames))))
-    elif len(dnsnames) == 1:
-        raise CertificateError("hostname %r "
-            "doesn't match %r"
-            % (hostname, dnsnames[0]))
-    else:
-        raise CertificateError("no appropriate commonName or "
-            "subjectAltName fields were found")
-
-
-DefaultVerifyPaths = namedtuple("DefaultVerifyPaths",
-    "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env "
-    "openssl_capath")
-
-def get_default_verify_paths():
-    """Return paths to default cafile and capath.
-    """
-    parts = _ssl.get_default_verify_paths()
-
-    # environment vars shadow paths
-    cafile = os.environ.get(parts[0], parts[1])
-    capath = os.environ.get(parts[2], parts[3])
-
-    return DefaultVerifyPaths(cafile if os.path.isfile(cafile) else None,
-                              capath if os.path.isdir(capath) else None,
-                              *parts)
-
-
-class _ASN1Object(namedtuple("_ASN1Object", "nid shortname longname oid")):
-    """ASN.1 object identifier lookup
-    """
-    __slots__ = ()
-
-    def __new__(cls, oid):
-        return super().__new__(cls, *_txt2obj(oid, name=False))
-
-    @classmethod
-    def fromnid(cls, nid):
-        """Create _ASN1Object from OpenSSL numeric ID
-        """
-        return super().__new__(cls, *_nid2obj(nid))
-
-    @classmethod
-    def fromname(cls, name):
-        """Create _ASN1Object from short name, long name or OID
-        """
-        return super().__new__(cls, *_txt2obj(name, name=True))
-
-
-class Purpose(_ASN1Object, _Enum):
-    """SSLContext purpose flags with X509v3 Extended Key Usage objects
-    """
-    SERVER_AUTH = '1.3.6.1.5.5.7.3.1'
-    CLIENT_AUTH = '1.3.6.1.5.5.7.3.2'
-
-
-class SSLContext(_SSLContext):
-    """An SSLContext holds various SSL-related configuration options and
-    data, such as certificates and possibly a private key."""
-
-    __slots__ = ('protocol', '__weakref__')
-    _windows_cert_stores = ("CA", "ROOT")
-
-    def __new__(cls, protocol, *args, **kwargs):
-        self = _SSLContext.__new__(cls, protocol)
-        if protocol != _SSLv2_IF_EXISTS:
-            self.set_ciphers(_DEFAULT_CIPHERS)
-        return self
-
-    def __init__(self, protocol):
-        self.protocol = protocol
-
-    def wrap_socket(self, sock, server_side=False,
-                    do_handshake_on_connect=True,
-                    suppress_ragged_eofs=True,
-                    server_hostname=None):
-        return SSLSocket(sock=sock, server_side=server_side,
-                         do_handshake_on_connect=do_handshake_on_connect,
-                         suppress_ragged_eofs=suppress_ragged_eofs,
-                         server_hostname=server_hostname,
-                         _context=self)
-
-    def wrap_bio(self, incoming, outgoing, server_side=False,
-                 server_hostname=None):
-        sslobj = self._wrap_bio(incoming, outgoing, server_side=server_side,
-                                server_hostname=server_hostname)
-        return SSLObject(sslobj)
-
-    def set_npn_protocols(self, npn_protocols):
-        protos = bytearray()
-        for protocol in npn_protocols:
-            b = bytes(protocol, 'ascii')
-            if len(b) == 0 or len(b) > 255:
-                raise SSLError('NPN protocols must be 1 to 255 in length')
-            protos.append(len(b))
-            protos.extend(b)
-
-        self._set_npn_protocols(protos)
-
-    def set_alpn_protocols(self, alpn_protocols):
-        protos = bytearray()
-        for protocol in alpn_protocols:
-            b = bytes(protocol, 'ascii')
-            if len(b) == 0 or len(b) > 255:
-                raise SSLError('ALPN protocols must be 1 to 255 in length')
-            protos.append(len(b))
-            protos.extend(b)
-
-        self._set_alpn_protocols(protos)
-
-    def _load_windows_store_certs(self, storename, purpose):
-        certs = bytearray()
-        try:
-            for cert, encoding, trust in enum_certificates(storename):
-                # CA certs are never PKCS#7 encoded
-                if encoding == "x509_asn":
-                    if trust is True or purpose.oid in trust:
-                        certs.extend(cert)
-        except PermissionError:
-            warnings.warn("unable to enumerate Windows certificate store")
-        if certs:
-            self.load_verify_locations(cadata=certs)
-        return certs
-
-    def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
-        if not isinstance(purpose, _ASN1Object):
-            raise TypeError(purpose)
-        if sys.platform == "win32":
-            for storename in self._windows_cert_stores:
-                self._load_windows_store_certs(storename, purpose)
-        self.set_default_verify_paths()
-
-
-def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None,
-                           capath=None, cadata=None):
-    """Create a SSLContext object with default settings.
-
-    NOTE: The protocol and settings may change anytime without prior
-          deprecation. The values represent a fair balance between maximum
-          compatibility and security.
-    """
-    if not isinstance(purpose, _ASN1Object):
-        raise TypeError(purpose)
-
-    context = SSLContext(PROTOCOL_SSLv23)
-
-    # SSLv2 considered harmful.
-    context.options |= OP_NO_SSLv2
-
-    # SSLv3 has problematic security and is only required for really old
-    # clients such as IE6 on Windows XP
-    context.options |= OP_NO_SSLv3
-
-    # disable compression to prevent CRIME attacks (OpenSSL 1.0+)
-    context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
-
-    if purpose == Purpose.SERVER_AUTH:
-        # verify certs and host name in client mode
-        context.verify_mode = CERT_REQUIRED
-        context.check_hostname = True
-    elif purpose == Purpose.CLIENT_AUTH:
-        # Prefer the server's ciphers by default so that we get stronger
-        # encryption
-        context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0)
-
-        # Use single use keys in order to improve forward secrecy
-        context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0)
-        context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0)
-
-        # disallow ciphers with known vulnerabilities
-        context.set_ciphers(_RESTRICTED_SERVER_CIPHERS)
-
-    if cafile or capath or cadata:
-        context.load_verify_locations(cafile, capath, cadata)
-    elif context.verify_mode != CERT_NONE:
-        # no explicit cafile, capath or cadata but the verify mode is
-        # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
-        # root CA certificates for the given purpose. This may fail silently.
-        context.load_default_certs(purpose)
-    return context
-
-def _create_unverified_context(protocol=PROTOCOL_SSLv23, *, cert_reqs=None,
-                           check_hostname=False, purpose=Purpose.SERVER_AUTH,
-                           certfile=None, keyfile=None,
-                           cafile=None, capath=None, cadata=None):
-    """Create a SSLContext object for Python stdlib modules
-
-    All Python stdlib modules shall use this function to create SSLContext
-    objects in order to keep common settings in one place. The configuration
-    is less restrict than create_default_context()'s to increase backward
-    compatibility.
-    """
-    if not isinstance(purpose, _ASN1Object):
-        raise TypeError(purpose)
-
-    context = SSLContext(protocol)
-    # SSLv2 considered harmful.
-    context.options |= OP_NO_SSLv2
-    # SSLv3 has problematic security and is only required for really old
-    # clients such as IE6 on Windows XP
-    context.options |= OP_NO_SSLv3
-
-    if cert_reqs is not None:
-        context.verify_mode = cert_reqs
-    context.check_hostname = check_hostname
-
-    if keyfile and not certfile:
-        raise ValueError("certfile must be specified")
-    if certfile or keyfile:
-        context.load_cert_chain(certfile, keyfile)
-
-    # load CA root certs
-    if cafile or capath or cadata:
-        context.load_verify_locations(cafile, capath, cadata)
-    elif context.verify_mode != CERT_NONE:
-        # no explicit cafile, capath or cadata but the verify mode is
-        # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
-        # root CA certificates for the given purpose. This may fail silently.
-        context.load_default_certs(purpose)
-
-    return context
-
-# Used by http.client if no context is explicitly passed.
-_create_default_https_context = create_default_context
-
-
-# Backwards compatibility alias, even though it's not a public name.
-_create_stdlib_context = _create_unverified_context
-
-
-class SSLObject:
-    """This class implements an interface on top of a low-level SSL object as
-    implemented by OpenSSL. This object captures the state of an SSL connection
-    but does not provide any network IO itself. IO needs to be performed
-    through separate "BIO" objects which are OpenSSL's IO abstraction layer.
-
-    This class does not have a public constructor. Instances are returned by
-    ``SSLContext.wrap_bio``. This class is typically used by framework authors
-    that want to implement asynchronous IO for SSL through memory buffers.
-
-    When compared to ``SSLSocket``, this object lacks the following features:
-
-     * Any form of network IO incluging methods such as ``recv`` and ``send``.
-     * The ``do_handshake_on_connect`` and ``suppress_ragged_eofs`` machinery.
-    """
-
-    def __init__(self, sslobj, owner=None):
-        self._sslobj = sslobj
-        # Note: _sslobj takes a weak reference to owner
-        self._sslobj.owner = owner or self
-
-    @property
-    def context(self):
-        """The SSLContext that is currently in use."""
-        return self._sslobj.context
-
-    @context.setter
-    def context(self, ctx):
-        self._sslobj.context = ctx
-
-    @property
-    def server_side(self):
-        """Whether this is a server-side socket."""
-        return self._sslobj.server_side
-
-    @property
-    def server_hostname(self):
-        """The currently set server hostname (for SNI), or ``None`` if no
-        server hostame is set."""
-        return self._sslobj.server_hostname
-
-    def read(self, len=1024, buffer=None):
-        """Read up to 'len' bytes from the SSL object and return them.
-
-        If 'buffer' is provided, read into this buffer and return the number of
-        bytes read.
-        """
-        if buffer is not None:
-            v = self._sslobj.read(len, buffer)
-        else:
-            v = self._sslobj.read(len)
-        return v
-
-    def write(self, data):
-        """Write 'data' to the SSL object and return the number of bytes
-        written.
-
-        The 'data' argument must support the buffer interface.
-        """
-        return self._sslobj.write(data)
-
-    def getpeercert(self, binary_form=False):
-        """Returns a formatted version of the data in the certificate provided
-        by the other end of the SSL channel.
-
-        Return None if no certificate was provided, {} if a certificate was
-        provided, but not validated.
-        """
-        return self._sslobj.peer_certificate(binary_form)
-
-    def selected_npn_protocol(self):
-        """Return the currently selected NPN protocol as a string, or ``None``
-        if a next protocol was not negotiated or if NPN is not supported by one
-        of the peers."""
-        if _ssl.HAS_NPN:
-            return self._sslobj.selected_npn_protocol()
-
-    def selected_alpn_protocol(self):
-        """Return the currently selected ALPN protocol as a string, or ``None``
-        if a next protocol was not negotiated or if ALPN is not supported by one
-        of the peers."""
-        if _ssl.HAS_ALPN:
-            return self._sslobj.selected_alpn_protocol()
-
-    def cipher(self):
-        """Return the currently selected cipher as a 3-tuple ``(name,
-        ssl_version, secret_bits)``."""
-        return self._sslobj.cipher()
-
-    def shared_ciphers(self):
-        """Return a list of ciphers shared by the client during the handshake or
-        None if this is not a valid server connection.
-        """
-        return self._sslobj.shared_ciphers()
-
-    def compression(self):
-        """Return the current compression algorithm in use, or ``None`` if
-        compression was not negotiated or not supported by one of the peers."""
-        return self._sslobj.compression()
-
-    def pending(self):
-        """Return the number of bytes that can be read immediately."""
-        return self._sslobj.pending()
-
-    def do_handshake(self):
-        """Start the SSL/TLS handshake."""
-        self._sslobj.do_handshake()
-        if self.context.check_hostname:
-            if not self.server_hostname:
-                raise ValueError("check_hostname needs server_hostname "
-                                 "argument")
-            match_hostname(self.getpeercert(), self.server_hostname)
-
-    def unwrap(self):
-        """Start the SSL shutdown handshake."""
-        return self._sslobj.shutdown()
-
-    def get_channel_binding(self, cb_type="tls-unique"):
-        """Get channel binding data for current connection.  Raise ValueError
-        if the requested `cb_type` is not supported.  Return bytes of the data
-        or None if the data is not available (e.g. before the handshake)."""
-        if cb_type not in CHANNEL_BINDING_TYPES:
-            raise ValueError("Unsupported channel binding type")
-        if cb_type != "tls-unique":
-            raise NotImplementedError(
-                            "{0} channel binding type not implemented"
-                            .format(cb_type))
-        return self._sslobj.tls_unique_cb()
-
-    def version(self):
-        """Return a string identifying the protocol version used by the
-        current SSL channel. """
-        return self._sslobj.version()
-
-
-class SSLSocket(socket):
-    """This class implements a subtype of socket.socket that wraps
-    the underlying OS socket in an SSL context when necessary, and
-    provides read and write methods over that channel."""
-
-    def __init__(self, sock=None, keyfile=None, certfile=None,
-                 server_side=False, cert_reqs=CERT_NONE,
-                 ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-                 do_handshake_on_connect=True,
-                 family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None,
-                 suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
-                 server_hostname=None,
-                 _context=None):
-
-        if _context:
-            self._context = _context
-        else:
-            if server_side and not certfile:
-                raise ValueError("certfile must be specified for server-side "
-                                 "operations")
-            if keyfile and not certfile:
-                raise ValueError("certfile must be specified")
-            if certfile and not keyfile:
-                keyfile = certfile
-            self._context = SSLContext(ssl_version)
-            self._context.verify_mode = cert_reqs
-            if ca_certs:
-                self._context.load_verify_locations(ca_certs)
-            if certfile:
-                self._context.load_cert_chain(certfile, keyfile)
-            if npn_protocols:
-                self._context.set_npn_protocols(npn_protocols)
-            if ciphers:
-                self._context.set_ciphers(ciphers)
-            self.keyfile = keyfile
-            self.certfile = certfile
-            self.cert_reqs = cert_reqs
-            self.ssl_version = ssl_version
-            self.ca_certs = ca_certs
-            self.ciphers = ciphers
-        # Can't use sock.type as other flags (such as SOCK_NONBLOCK) get
-        # mixed in.
-        if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM:
-            raise NotImplementedError("only stream sockets are supported")
-        if server_side and server_hostname:
-            raise ValueError("server_hostname can only be specified "
-                             "in client mode")
-        if self._context.check_hostname and not server_hostname:
-            raise ValueError("check_hostname requires server_hostname")
-        self.server_side = server_side
-        self.server_hostname = server_hostname
-        self.do_handshake_on_connect = do_handshake_on_connect
-        self.suppress_ragged_eofs = suppress_ragged_eofs
-        if sock is not None:
-            socket.__init__(self,
-                            family=sock.family,
-                            type=sock.type,
-                            proto=sock.proto,
-                            fileno=sock.fileno())
-            self.settimeout(sock.gettimeout())
-            sock.detach()
-        elif fileno is not None:
-            socket.__init__(self, fileno=fileno)
-        else:
-            socket.__init__(self, family=family, type=type, proto=proto)
-
-        # See if we are connected
-        try:
-            self.getpeername()
-        except OSError as e:
-            if e.errno != errno.ENOTCONN:
-                raise
-            connected = False
-        else:
-            connected = True
-
-        self._closed = False
-        self._sslobj = None
-        self._connected = connected
-        if connected:
-            # create the SSL object
-            try:
-                sslobj = self._context._wrap_socket(self, server_side,
-                                                    server_hostname)
-                self._sslobj = SSLObject(sslobj, owner=self)
-                if do_handshake_on_connect:
-                    timeout = self.gettimeout()
-                    if timeout == 0.0:
-                        # non-blocking
-                        raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-                    self.do_handshake()
-
-            except (OSError, ValueError):
-                self.close()
-                raise
-
-    @property
-    def context(self):
-        return self._context
-
-    @context.setter
-    def context(self, ctx):
-        self._context = ctx
-        self._sslobj.context = ctx
-
-    def dup(self):
-        raise NotImplemented("Can't dup() %s instances" %
-                             self.__class__.__name__)
-
-    def _checkClosed(self, msg=None):
-        # raise an exception here if you wish to check for spurious closes
-        pass
-
-    def _check_connected(self):
-        if not self._connected:
-            # getpeername() will raise ENOTCONN if the socket is really
-            # not connected; note that we can be connected even without
-            # _connected being set, e.g. if connect() first returned
-            # EAGAIN.
-            self.getpeername()
-
-    def read(self, len=1024, buffer=None):
-        """Read up to LEN bytes and return them.
-        Return zero-length string on EOF."""
-
-        self._checkClosed()
-        if not self._sslobj:
-            raise ValueError("Read on closed or unwrapped SSL socket.")
-        try:
-            return self._sslobj.read(len, buffer)
-        except SSLError as x:
-            if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:
-                if buffer is not None:
-                    return 0
-                else:
-                    return b''
-            else:
-                raise
-
-    def write(self, data):
-        """Write DATA to the underlying SSL channel.  Returns
-        number of bytes of DATA actually transmitted."""
-
-        self._checkClosed()
-        if not self._sslobj:
-            raise ValueError("Write on closed or unwrapped SSL socket.")
-        return self._sslobj.write(data)
-
-    def getpeercert(self, binary_form=False):
-        """Returns a formatted version of the data in the
-        certificate provided by the other end of the SSL channel.
-        Return None if no certificate was provided, {} if a
-        certificate was provided, but not validated."""
-
-        self._checkClosed()
-        self._check_connected()
-        return self._sslobj.getpeercert(binary_form)
-
-    def selected_npn_protocol(self):
-        self._checkClosed()
-        if not self._sslobj or not _ssl.HAS_NPN:
-            return None
-        else:
-            return self._sslobj.selected_npn_protocol()
-
-    def selected_alpn_protocol(self):
-        self._checkClosed()
-        if not self._sslobj or not _ssl.HAS_ALPN:
-            return None
-        else:
-            return self._sslobj.selected_alpn_protocol()
-
-    def cipher(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        else:
-            return self._sslobj.cipher()
-
-    def shared_ciphers(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        return self._sslobj.shared_ciphers()
-
-    def compression(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        else:
-            return self._sslobj.compression()
-
-    def send(self, data, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to send() on %s" %
-                    self.__class__)
-            return self._sslobj.write(data)
-        else:
-            return socket.send(self, data, flags)
-
-    def sendto(self, data, flags_or_addr, addr=None):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("sendto not allowed on instances of %s" %
-                             self.__class__)
-        elif addr is None:
-            return socket.sendto(self, data, flags_or_addr)
-        else:
-            return socket.sendto(self, data, flags_or_addr, addr)
-
-    def sendmsg(self, *args, **kwargs):
-        # Ensure programs don't send data unencrypted if they try to
-        # use this method.
-        raise NotImplementedError("sendmsg not allowed on instances of %s" %
-                                  self.__class__)
-
-    def sendall(self, data, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to sendall() on %s" %
-                    self.__class__)
-            amount = len(data)
-            count = 0
-            while (count < amount):
-                v = self.send(data[count:])
-                count += v
-            return amount
-        else:
-            return socket.sendall(self, data, flags)
-
-    def sendfile(self, file, offset=0, count=None):
-        """Send a file, possibly by using os.sendfile() if this is a
-        clear-text socket.  Return the total number of bytes sent.
-        """
-        if self._sslobj is None:
-            # os.sendfile() works with plain sockets only
-            return super().sendfile(file, offset, count)
-        else:
-            return self._sendfile_use_send(file, offset, count)
-
-    def recv(self, buflen=1024, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to recv() on %s" %
-                    self.__class__)
-            return self.read(buflen)
-        else:
-            return socket.recv(self, buflen, flags)
-
-    def recv_into(self, buffer, nbytes=None, flags=0):
-        self._checkClosed()
-        if buffer and (nbytes is None):
-            nbytes = len(buffer)
-        elif nbytes is None:
-            nbytes = 1024
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                  "non-zero flags not allowed in calls to recv_into() on %s" %
-                  self.__class__)
-            return self.read(nbytes, buffer)
-        else:
-            return socket.recv_into(self, buffer, nbytes, flags)
-
-    def recvfrom(self, buflen=1024, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("recvfrom not allowed on instances of %s" %
-                             self.__class__)
-        else:
-            return socket.recvfrom(self, buflen, flags)
-
-    def recvfrom_into(self, buffer, nbytes=None, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("recvfrom_into not allowed on instances of %s" %
-                             self.__class__)
-        else:
-            return socket.recvfrom_into(self, buffer, nbytes, flags)
-
-    def recvmsg(self, *args, **kwargs):
-        raise NotImplementedError("recvmsg not allowed on instances of %s" %
-                                  self.__class__)
-
-    def recvmsg_into(self, *args, **kwargs):
-        raise NotImplementedError("recvmsg_into not allowed on instances of "
-                                  "%s" % self.__class__)
-
-    def pending(self):
-        self._checkClosed()
-        if self._sslobj:
-            return self._sslobj.pending()
-        else:
-            return 0
-
-    def shutdown(self, how):
-        self._checkClosed()
-        self._sslobj = None
-        socket.shutdown(self, how)
-
-    def unwrap(self):
-        if self._sslobj:
-            s = self._sslobj.unwrap()
-            self._sslobj = None
-            return s
-        else:
-            raise ValueError("No SSL wrapper around " + str(self))
-
-    def _real_close(self):
-        self._sslobj = None
-        socket._real_close(self)
-
-    def do_handshake(self, block=False):
-        """Perform a TLS/SSL handshake."""
-        self._check_connected()
-        timeout = self.gettimeout()
-        try:
-            if timeout == 0.0 and block:
-                self.settimeout(None)
-            self._sslobj.do_handshake()
-        finally:
-            self.settimeout(timeout)
-
-    def _real_connect(self, addr, connect_ex):
-        if self.server_side:
-            raise ValueError("can't connect in server-side mode")
-        # Here we assume that the socket is client-side, and not
-        # connected at the time of the call.  We connect it, then wrap it.
-        if self._connected:
-            raise ValueError("attempt to connect already-connected SSLSocket!")
-        sslobj = self.context._wrap_socket(self, False, self.server_hostname)
-        self._sslobj = SSLObject(sslobj, owner=self)
-        try:
-            if connect_ex:
-                rc = socket.connect_ex(self, addr)
-            else:
-                rc = None
-                socket.connect(self, addr)
-            if not rc:
-                self._connected = True
-                if self.do_handshake_on_connect:
-                    self.do_handshake()
-            return rc
-        except (OSError, ValueError):
-            self._sslobj = None
-            raise
-
-    def connect(self, addr):
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-        self._real_connect(addr, False)
-
-    def connect_ex(self, addr):
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-        return self._real_connect(addr, True)
-
-    def accept(self):
-        """Accepts a new connection from a remote client, and returns
-        a tuple containing that new connection wrapped with a server-side
-        SSL channel, and the address of the remote client."""
-
-        newsock, addr = socket.accept(self)
-        newsock = self.context.wrap_socket(newsock,
-                    do_handshake_on_connect=self.do_handshake_on_connect,
-                    suppress_ragged_eofs=self.suppress_ragged_eofs,
-                    server_side=True)
-        return newsock, addr
-
-    def get_channel_binding(self, cb_type="tls-unique"):
-        """Get channel binding data for current connection.  Raise ValueError
-        if the requested `cb_type` is not supported.  Return bytes of the data
-        or None if the data is not available (e.g. before the handshake).
-        """
-        if self._sslobj is None:
-            return None
-        return self._sslobj.get_channel_binding(cb_type)
-
-    def version(self):
-        """
-        Return a string identifying the protocol version used by the
-        current SSL channel, or None if there is no established channel.
-        """
-        if self._sslobj is None:
-            return None
-        return self._sslobj.version()
-
-
-def wrap_socket(sock, keyfile=None, certfile=None,
-                server_side=False, cert_reqs=CERT_NONE,
-                ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-                do_handshake_on_connect=True,
-                suppress_ragged_eofs=True,
-                ciphers=None):
-
-    return SSLSocket(sock=sock, keyfile=keyfile, certfile=certfile,
-                     server_side=server_side, cert_reqs=cert_reqs,
-                     ssl_version=ssl_version, ca_certs=ca_certs,
-                     do_handshake_on_connect=do_handshake_on_connect,
-                     suppress_ragged_eofs=suppress_ragged_eofs,
-                     ciphers=ciphers)
-
-# some utility functions
-
-def cert_time_to_seconds(cert_time):
-    """Return the time in seconds since the Epoch, given the timestring
-    representing the "notBefore" or "notAfter" date from a certificate
-    in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale).
-
-    "notBefore" or "notAfter" dates must use UTC (RFC 5280).
-
-    Month is one of: Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec
-    UTC should be specified as GMT (see ASN1_TIME_print())
-    """
-    from time import strptime
-    from calendar import timegm
-
-    months = (
-        "Jan","Feb","Mar","Apr","May","Jun",
-        "Jul","Aug","Sep","Oct","Nov","Dec"
-    )
-    time_format = ' %d %H:%M:%S %Y GMT' # NOTE: no month, fixed GMT
-    try:
-        month_number = months.index(cert_time[:3].title()) + 1
-    except ValueError:
-        raise ValueError('time data %r does not match '
-                         'format "%%b%s"' % (cert_time, time_format))
-    else:
-        # found valid month
-        tt = strptime(cert_time[3:], time_format)
-        # return an integer, the previous mktime()-based implementation
-        # returned a float (fractional seconds are always zero here).
-        return timegm((tt[0], month_number) + tt[2:6])
-
-PEM_HEADER = "-----BEGIN CERTIFICATE-----"
-PEM_FOOTER = "-----END CERTIFICATE-----"
-
-def DER_cert_to_PEM_cert(der_cert_bytes):
-    """Takes a certificate in binary DER format and returns the
-    PEM version of it as a string."""
-
-    f = str(base64.standard_b64encode(der_cert_bytes), 'ASCII', 'strict')
-    return (PEM_HEADER + '\n' +
-            textwrap.fill(f, 64) + '\n' +
-            PEM_FOOTER + '\n')
-
-def PEM_cert_to_DER_cert(pem_cert_string):
-    """Takes a certificate in ASCII PEM format and returns the
-    DER-encoded version of it as a byte sequence"""
-
-    if not pem_cert_string.startswith(PEM_HEADER):
-        raise ValueError("Invalid PEM encoding; must start with %s"
-                         % PEM_HEADER)
-    if not pem_cert_string.strip().endswith(PEM_FOOTER):
-        raise ValueError("Invalid PEM encoding; must end with %s"
-                         % PEM_FOOTER)
-    d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)]
-    return base64.decodebytes(d.encode('ASCII', 'strict'))
-
-def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None):
-    """Retrieve the certificate from the server at the specified address,
-    and return it as a PEM-encoded string.
-    If 'ca_certs' is specified, validate the server cert against it.
-    If 'ssl_version' is specified, use it in the connection attempt."""
-
-    host, port = addr
-    if ca_certs is not None:
-        cert_reqs = CERT_REQUIRED
-    else:
-        cert_reqs = CERT_NONE
-    context = _create_stdlib_context(ssl_version,
-                                     cert_reqs=cert_reqs,
-                                     cafile=ca_certs)
-    with  create_connection(addr) as sock:
-        with context.wrap_socket(sock) as sslsock:
-            dercert = sslsock.getpeercert(True)
-    return DER_cert_to_PEM_cert(dercert)
-
-def get_protocol_name(protocol_code):
-    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')

From pypy.commits at gmail.com  Mon Nov 28 09:15:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 06:15:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-multiprocessing: merge py3.5
Message-ID: <583c3bfc.c9b3c20a.40d2f.fa03@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-multiprocessing
Changeset: r88697:306ff51dda10
Date: 2016-11-28 11:48 +0100
http://bitbucket.org/pypy/pypy/changeset/306ff51dda10/

Log:	merge py3.5

diff too long, truncating to 2000 out of 18830 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,6 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
+.git/
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,18 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -118,30 +124,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (
diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -370,12 +370,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -428,15 +429,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -12,7 +12,7 @@
 
 import sys
 import os
-import shlex
+import imp
 
 from distutils.errors import DistutilsPlatformError
 
@@ -61,11 +61,19 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['SHLIB_SUFFIX'] = g['SO']
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = ".so"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
+    g['VERSION'] = get_python_version()
 
     global _config_vars
     _config_vars = g
@@ -76,7 +84,7 @@
     g = {}
     g['EXE'] = ".exe"
     g['SO'] = ".pyd"
-    g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['SOABI'] = g['SO'].rsplit('.')[0]   # xxx?
 
     global _config_vars
     _config_vars = g
@@ -118,26 +126,78 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile.
     """
     if compiler.compiler_type == "unix":
-        compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            compiler.compiler.extend(cppflags)
-            compiler.compiler_so.extend(cppflags)
-            compiler.linker_so.extend(cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            compiler.compiler.extend(cflags)
-            compiler.compiler_so.extend(cflags)
-            compiler.linker_so.extend(cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            compiler.linker_so.extend(ldflags)
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+        (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = shlib_suffix
 
 
 from .sysconfig_cpython import (
diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -145,6 +145,7 @@
 from socket import SOL_SOCKET, SO_TYPE
 import base64        # for DER-to-PEM translation
 import errno
+import warnings
 
 
 socket_error = OSError  # keep that public name in module namespace
@@ -405,12 +406,16 @@
 
     def _load_windows_store_certs(self, storename, purpose):
         certs = bytearray()
-        for cert, encoding, trust in enum_certificates(storename):
-            # CA certs are never PKCS#7 encoded
-            if encoding == "x509_asn":
-                if trust is True or purpose.oid in trust:
-                    certs.extend(cert)
-        self.load_verify_locations(cadata=certs)
+        try:
+            for cert, encoding, trust in enum_certificates(storename):
+                # CA certs are never PKCS#7 encoded
+                if encoding == "x509_asn":
+                    if trust is True or purpose.oid in trust:
+                        certs.extend(cert)
+        except PermissionError:
+            warnings.warn("unable to enumerate Windows certificate store")
+        if certs:
+            self.load_verify_locations(cadata=certs)
         return certs
 
     def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
@@ -560,7 +565,7 @@
         server hostame is set."""
         return self._sslobj.server_hostname
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to 'len' bytes from the SSL object and return them.
 
         If 'buffer' is provided, read into this buffer and return the number of
@@ -569,7 +574,7 @@
         if buffer is not None:
             v = self._sslobj.read(len, buffer)
         else:
-            v = self._sslobj.read(len or 1024)
+            v = self._sslobj.read(len)
         return v
 
     def write(self, data):
@@ -745,7 +750,8 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-            except:
+
+            except (OSError, ValueError):
                 self.close()
                 raise
 
@@ -774,7 +780,7 @@
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
 
diff --git a/lib-python/3/test/capath/0e4015b9.0 b/lib-python/3/test/capath/0e4015b9.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/0e4015b9.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/capath/ce7b8643.0 b/lib-python/3/test/capath/ce7b8643.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/ce7b8643.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/selfsigned_pythontestdotnet.pem b/lib-python/3/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/3/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/3/test/selfsigned_pythontestdotnet.pem
@@ -1,5 +1,5 @@
 -----BEGIN CERTIFICATE-----
-MIIChzCCAfCgAwIBAgIJAKGU95wKR8pSMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
 IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
 bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
@@ -8,9 +8,9 @@
 aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
 Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
 Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjKTAnMCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MA0GCSqGSIb3DQEBBQUAA4GBAIOXmdtM
-eG9qzP9TiXW/Gc/zI4cBfdCpC+Y4gOfC9bQUC7hefix4iO3+iZjgy3X/FaRxUUoV
-HKiXcXIaWqTSUWp45cSh0MbwZXudp6JIAptzdAhvvCrPKeC9i9GvxsPD4LtDAL97
-vSaxQBezA7hdxZd90/EeyMgVZgAnTCnvAWX9
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
 -----END CERTIFICATE-----
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -55,11 +55,12 @@
 SIGNED_CERTFILE2 = data_file("keycert4.pem")
 SIGNING_CA = data_file("pycacert.pem")
 
-SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem")
+REMOTE_HOST = "self-signed.pythontest.net"
+REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem")
 
 EMPTYCERT = data_file("nullcert.pem")
 BADCERT = data_file("badcert.pem")
-WRONGCERT = data_file("XXXnonexisting.pem")
+NONEXISTINGCERT = data_file("XXXnonexisting.pem")
 BADKEY = data_file("badkey.pem")
 NOKIACERT = data_file("nokia.pem")
 NULLBYTECERT = data_file("nullbytecert.pem")
@@ -276,7 +277,7 @@
         self.assertEqual(p['subjectAltName'], san)
 
     def test_DER_to_PEM(self):
-        with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f:
+        with open(CAFILE_CACERT, 'r') as f:
             pem = f.read()
         d1 = ssl.PEM_cert_to_DER_cert(pem)
         p2 = ssl.DER_cert_to_PEM_cert(d1)
@@ -366,17 +367,42 @@
                                     s.connect, (HOST, 8080))
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT)
+                ssl.wrap_socket(sock, certfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=CERTFILE, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def bad_cert_test(self, certfile):
+        """Check that trying to use the given client certificate fails"""
+        certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                   certfile)
+        sock = socket.socket()
+        self.addCleanup(sock.close)
+        with self.assertRaises(ssl.SSLError):
+            ssl.wrap_socket(sock,
+                            certfile=certfile,
+                            ssl_version=ssl.PROTOCOL_TLSv1)
+
+    def test_empty_cert(self):
+        """Wrapping with an empty cert file"""
+        self.bad_cert_test("nullcert.pem")
+
+    def test_malformed_cert(self):
+        """Wrapping with a badly formatted certificate (syntax error)"""
+        self.bad_cert_test("badcert.pem")
+
+    def test_malformed_key(self):
+        """Wrapping with a badly formatted key (syntax error)"""
+        self.bad_cert_test("badkey.pem")
+
     def test_match_hostname(self):
         def ok(cert, hostname):
             ssl.match_hostname(cert, hostname)
@@ -795,7 +821,8 @@
             self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
                              ctx.options)
             ctx.options = 0
-            self.assertEqual(0, ctx.options)
+            # Ubuntu has OP_NO_SSLv3 forced on by default
+            self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3)
         else:
             with self.assertRaises(ValueError):
                 ctx.options = 0
@@ -842,7 +869,7 @@
         ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE)
         self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE)
         with self.assertRaises(OSError) as cm:
-            ctx.load_cert_chain(WRONGCERT)
+            ctx.load_cert_chain(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_cert_chain(BADCERT)
@@ -862,7 +889,7 @@
         # Mismatching key and cert
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"):
-            ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY)
+            ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY)
         # Password protected key and cert
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD)
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode())
@@ -927,7 +954,7 @@
         self.assertRaises(TypeError, ctx.load_verify_locations)
         self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None)
         with self.assertRaises(OSError) as cm:
-            ctx.load_verify_locations(WRONGCERT)
+            ctx.load_verify_locations(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_verify_locations(BADCERT)
@@ -1003,7 +1030,7 @@
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
         with self.assertRaises(FileNotFoundError) as cm:
-            ctx.load_dh_params(WRONGCERT)
+            ctx.load_dh_params(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(ssl.SSLError) as cm:
             ctx.load_dh_params(CERTFILE)
@@ -1080,7 +1107,7 @@
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 0, 'crl': 0, 'x509': 1})
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 1, 'crl': 0, 'x509': 2})
 
@@ -1090,8 +1117,8 @@
         # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.get_ca_certs(), [])
-        # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        # but CAFILE_CACERT is a CA cert
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.get_ca_certs(),
             [{'issuer': ((('organizationName', 'Root CA'),),
                          (('organizationalUnitName', 'http://www.cacert.org'),),
@@ -1107,7 +1134,7 @@
                           (('emailAddress', 'support at cacert.org'),)),
               'version': 3}])
 
-        with open(SVN_PYTHON_ORG_ROOT_CERT) as f:
+        with open(CAFILE_CACERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
         self.assertEqual(ctx.get_ca_certs(True), [der])
@@ -1345,11 +1372,11 @@
 class NetworkedTests(unittest.TestCase):
 
     def test_connect(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
@@ -1358,27 +1385,27 @@
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED)
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                   s.connect, ("svn.python.org", 443))
+                                   s.connect, (REMOTE_HOST, 443))
             s.close()
 
             # this should succeed because we specify the root cert
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
 
     def test_connect_ex(self):
         # Issue #11326: check connect_ex() implementation
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                self.assertEqual(0, s.connect_ex(("svn.python.org", 443)))
+                self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443)))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
@@ -1386,14 +1413,14 @@
     def test_non_blocking_connect_ex(self):
         # Issue #11326: non-blocking connect_ex() should allow handshake
         # to proceed after the socket gets ready.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.setblocking(False)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
                 self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
                 # Wait for connect to finish
@@ -1415,58 +1442,62 @@
     def test_timeout_connect_ex(self):
         # Issue #12065: on a timeout, connect_ex() should return the original
         # errno (mimicking the behaviour of non-SSL sockets).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.settimeout(0.0000001)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 if rc == 0:
-                    self.skipTest("svn.python.org responded too quickly")
+                    self.skipTest("REMOTE_HOST responded too quickly")
                 self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK))
             finally:
                 s.close()
 
     def test_connect_ex_error(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                rc = s.connect_ex(("svn.python.org", 444))
+                rc = s.connect_ex((REMOTE_HOST, 444))
                 # Issue #19919: Windows machines or VMs hosted on Windows
                 # machines sometimes return EWOULDBLOCK.
-                self.assertIn(rc, (errno.ECONNREFUSED, errno.EWOULDBLOCK))
+                errors = (
+                    errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT,
+                    errno.EWOULDBLOCK,
+                )
+                self.assertIn(rc, errors)
             finally:
                 s.close()
 
     def test_connect_with_context(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             # Same as test_connect, but with a separately created context
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
             # Same with a server hostname
             s = ctx.wrap_socket(socket.socket(socket.AF_INET),
-                                server_hostname="svn.python.org")
-            s.connect(("svn.python.org", 443))
+                                server_hostname=REMOTE_HOST)
+            s.connect((REMOTE_HOST, 443))
             s.close()
             # This should fail because we have no verification certs
             ctx.verify_mode = ssl.CERT_REQUIRED
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                    s.connect, ("svn.python.org", 443))
+                                    s.connect, (REMOTE_HOST, 443))
             s.close()
             # This should succeed because we specify the root cert
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1479,12 +1510,12 @@
         # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must
         # contain both versions of each certificate (same content, different
         # filename) for this test to be portable across OpenSSL releases.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1495,7 +1526,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=BYTES_CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1503,15 +1534,15 @@
                 s.close()
 
     def test_connect_cadata(self):
-        with open(CAFILE_CACERT) as f:
+        with open(REMOTE_ROOT_CERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=pem)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1520,7 +1551,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=der)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1529,9 +1560,9 @@
         # Issue #5238: creating a file-like object with makefile() shouldn't
         # delay closing the underlying "real socket" (here tested with its
         # file descriptor, hence skipping the test under Windows).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
-            ss.connect(("svn.python.org", 443))
+            ss.connect((REMOTE_HOST, 443))
             fd = ss.fileno()
             f = ss.makefile()
             f.close()
@@ -1545,9 +1576,9 @@
             self.assertEqual(e.exception.errno, errno.EBADF)
 
     def test_non_blocking_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = socket.socket(socket.AF_INET)
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             s.setblocking(False)
             s = ssl.wrap_socket(s,
                                 cert_reqs=ssl.CERT_NONE,
@@ -1590,12 +1621,12 @@
                 if support.verbose:
                     sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))
 
-        _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT)
+        _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT)
         if support.IPV6_ENABLED:
             _test_get_server_certificate('ipv6.google.com', 443)
 
     def test_ciphers(self):
-        remote = ("svn.python.org", 443)
+        remote = (REMOTE_HOST, 443)
         with support.transient_internet(remote[0]):
             with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                  cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
@@ -1640,13 +1671,13 @@
 
     def test_get_ca_certs_capath(self):
         # capath certs are loaded on request
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             self.assertEqual(ctx.get_ca_certs(), [])
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1657,12 +1688,12 @@
     @needs_sni
     def test_context_setget(self):
         # Check that the context of a connected socket can be replaced.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = socket.socket(socket.AF_INET)
             with ctx1.wrap_socket(s) as ss:
-                ss.connect(("svn.python.org", 443))
+                ss.connect((REMOTE_HOST, 443))
                 self.assertIs(ss.context, ctx1)
                 self.assertIs(ss._sslobj.context, ctx1)
                 ss.context = ctx2
@@ -1683,13 +1714,8 @@
             try:
                 ret = func(*args)
             except ssl.SSLError as e:
-                # Note that we get a spurious -1/SSL_ERROR_SYSCALL for
-                # non-blocking IO. The SSL_shutdown manpage hints at this.
-                # It *should* be safe to just ignore SYS_ERROR_SYSCALL because
-                # with a Memory BIO there's no syscalls (for IO at least).
                 if e.errno not in (ssl.SSL_ERROR_WANT_READ,
-                                   ssl.SSL_ERROR_WANT_WRITE,
-                                   ssl.SSL_ERROR_SYSCALL):
+                                   ssl.SSL_ERROR_WANT_WRITE):
                     raise
                 errno = e.errno
             # Get any data from the outgoing BIO irrespective of any error, and
@@ -1712,36 +1738,46 @@
         return ret
 
     def test_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        # NOTE: this test has been modified, CPython in newer versions
+        # removed the ability to get the shared ciphers of the session, but
+        # they always return the cipher of the ssl context. This test is fully
+        # removed in later versions
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             ctx.check_hostname = True
-            sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org')
+            sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
-            self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            try:
+                self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            except ssl.SSLSyscallError:
+                # self-signed.pythontest.net probably shuts down the TCP
+                # connection without sending a secure shutdown message, and
+                # this is reported as SSL_ERROR_SYSCALL
+                pass
             self.assertRaises(ssl.SSLError, sslobj.write, b'foo')
             sock.close()
 
     def test_read_write_data(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -1838,14 +1874,7 @@
                 else:
                     self.sock.close()
 
-            # PyPy change
             def run(self):
-                try:
-                    self._run()
-                finally:
-                    self.close()
-
-            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
@@ -2091,36 +2120,6 @@
             self.active = False
             self.server.close()
 
-    def bad_cert_test(certfile):
-        """
-        Launch a server with CERT_REQUIRED, and check that trying to
-        connect to it with the given client certificate fails.
-        """
-        server = ThreadedEchoServer(CERTFILE,
-                                    certreqs=ssl.CERT_REQUIRED,
-                                    cacerts=CERTFILE, chatty=False,
-                                    connectionchatty=False)
-        with server:
-            try:
-                with socket.socket() as sock:
-                    s = ssl.wrap_socket(sock,
-                                        certfile=certfile,
-                                        ssl_version=ssl.PROTOCOL_TLSv1)
-                    s.connect((HOST, server.port))
-            except ssl.SSLError as x:
-                if support.verbose:
-                    sys.stdout.write("\nSSLError is %s\n" % x.args[1])
-            except OSError as x:
-                if support.verbose:
-                    sys.stdout.write("\nOSError is %s\n" % x.args[1])
-            except OSError as x:
-                if x.errno != errno.ENOENT:
-                    raise
-                if support.verbose:
-                    sys.stdout.write("\OSError is %s\n" % str(x))
-            else:
-                raise AssertionError("Use of invalid cert should have failed!")
-
     def server_params_test(client_context, server_context, indata=b"FOO\n",
                            chatty=True, connectionchatty=False, sni_name=None):
         """
@@ -2361,22 +2360,38 @@
                                                 "check_hostname requires server_hostname"):
                         context.wrap_socket(s)
 
-        def test_empty_cert(self):
-            """Connecting with an empty cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                      "nullcert.pem"))
-        def test_malformed_cert(self):
-            """Connecting with a badly formatted certificate (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badcert.pem"))
-        def test_nonexisting_cert(self):
-            """Connecting with a non-existing cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "wrongcert.pem"))
-        def test_malformed_key(self):
-            """Connecting with a badly formatted key (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badkey.pem"))
+        def test_wrong_cert(self):
+            """Connecting when the server rejects the client's certificate
+
+            Launch a server with CERT_REQUIRED, and check that trying to
+            connect to it with a wrong client certificate fails.
+            """
+            certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                       "wrongcert.pem")
+            server = ThreadedEchoServer(CERTFILE,
+                                        certreqs=ssl.CERT_REQUIRED,
+                                        cacerts=CERTFILE, chatty=False,
+                                        connectionchatty=False)
+            with server, \
+                    socket.socket() as sock, \
+                    ssl.wrap_socket(sock,
+                                        certfile=certfile,
+                                        ssl_version=ssl.PROTOCOL_TLSv1) as s:
+                try:
+                    # Expect either an SSL error about the server rejecting
+                    # the connection, or a low-level connection reset (which
+                    # sometimes happens on Windows)
+                    s.connect((HOST, server.port))
+                except ssl.SSLError as e:
+                    if support.verbose:
+                        sys.stdout.write("\nSSLError is %r\n" % e)
+                except OSError as e:
+                    if e.errno != errno.ECONNRESET:
+                        raise
+                    if support.verbose:
+                        sys.stdout.write("\nsocket.error is %r\n" % e)
+                else:
+                    self.fail("Use of invalid cert should have failed!")
 
         def test_rude_shutdown(self):
             """A brutal shutdown of an SSL server should raise an OSError
@@ -2782,6 +2797,20 @@
                         # consume data
                         s.read()
 
+                data = b"data"
+
+                # read(-1, buffer) is supported, even though read(-1) is not
+                s.send(data)
+                buffer = bytearray(len(data))
+                self.assertEqual(s.read(-1, buffer), len(data))
+                self.assertEqual(buffer, data)
+
+                # recv/read(0) should return no data
+                s.send(data)
+                self.assertEqual(s.recv(0), b"")
+                self.assertEqual(s.read(0), b"")
+                self.assertEqual(s.read(), data)
+
                 # Make sure sendmsg et al are disallowed to avoid
                 # inadvertent disclosure of data and/or corruption
                 # of the encrypted data stream
@@ -2791,6 +2820,10 @@
                                   s.recvmsg_into, bytearray(100))
 
                 s.write(b"over\n")
+
+                self.assertRaises(ValueError, s.recv, -1)
+                self.assertRaises(ValueError, s.read, -1)
+
                 s.close()
 
         def test_nonblocking_send(self):
@@ -3253,18 +3286,31 @@
             self.assertIn("TypeError", stderr.getvalue())
 
         def test_shared_ciphers(self):
+            # NOTE: This test case has been copied from master (~3.6)
+            # reason is the method SSL_get_ciphers (before that it used
+            # get_ciphers of the session which is not public)
             server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             server_context.load_cert_chain(SIGNED_CERTFILE)
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            client_context.set_ciphers("RC4")
-            server_context.set_ciphers("AES:RC4")
+            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
+                client_context.set_ciphers("AES128:AES256")
+                server_context.set_ciphers("AES256")
+                alg1 = "AES256"
+                alg2 = "AES-256"
+            else:
+                client_context.set_ciphers("AES:3DES")
+                server_context.set_ciphers("3DES")
+                alg1 = "3DES"
+                alg2 = "DES-CBC3"
+
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                self.assertIn("RC4", name.split("-"))
+                if not alg1 in name.split("-") and alg2 not in name:
+                    self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -3332,7 +3378,7 @@
             pass
 
     for filename in [
-        CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE,
+        CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE,
         ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY,
         SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA,
         BADCERT, BADKEY, EMPTYCERT]:
diff --git a/lib-python/3/test/wrongcert.pem b/lib-python/3/test/wrongcert.pem
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/wrongcert.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH
+FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T
+f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB
+AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq
+1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW
+7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg
+SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe
+19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg
+ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666
+lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs
+ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv
+frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk
+2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6
++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK
+24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G
+A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9
+OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt
+U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ
+fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E
+usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+
+43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw
+eSHj5jpC8iZKjCHBn+mAi4cQ514=
+-----END CERTIFICATE-----
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Vim
+
+*.swp
+*.swo
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,16 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..180c02ff82d3363f34a334aae22c9876d4c96481
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,85 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,

From pypy.commits at gmail.com  Mon Nov 28 09:32:16 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 06:32:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: remove review comment (created issue), 
Message-ID: <583c3ff0.e6b0c20a.f9dcd.8574@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88699:32c4efd0d776
Date: 2016-11-28 15:31 +0100
http://bitbucket.org/pypy/pypy/changeset/32c4efd0d776/

Log:	remove review comment (created issue), call SSL_free when the CData
	goes out of scope, ensure that the SSL peer certificate is collected

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -218,7 +218,7 @@
 
         lib.ERR_get_state()
         lib.ERR_clear_error()
-        self.ssl = ssl = lib.SSL_new(ctx)
+        self.ssl = ssl = lib.gc(lib.SSL_new(ctx), lib.SSL_free)
 
         self._app_data_handle = ffi.new_handle(self)
         lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
@@ -271,9 +271,6 @@
         self.server_hostname = None
         self.socket = None
 
-    def __del__(self):
-        self._app_data_handle = None
-
     @property
     def owner(self):
         if self._owner is None:
@@ -348,9 +345,7 @@
         if ret < 1:
             raise pyssl_error(self, ret)
 
-        if self.peer_cert != ffi.NULL:
-            lib.X509_free(self.peer_cert)
-        self.peer_cert = lib.SSL_get_peer_certificate(ssl)
+        self.peer_cert = lib.gc(lib.SSL_get_peer_certificate(ssl), lib.X509_free)
         #PySSL_END_ALLOW_THREADS
         self.handshake_done = 1
         return None
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -9,8 +9,6 @@
 
 def extension_suffixes(space):
     suffixes_w = []
-    # REVIEW this condition prevents pypy/tools/build_cffi_imports.py to succeed if
-    # pypy was compiled without cpyext module. why nore always append it?
     if space.config.objspace.usemodules.cpyext:
         suffixes_w.append(space.wrap(importing.get_so_extension(space)))
     return space.newlist(suffixes_w)

From pypy.commits at gmail.com  Mon Nov 28 09:36:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 06:36:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: ensure that the bio object is also
 deallocated
Message-ID: <583c40e7.8ab81c0a.d8a90.ce10@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88700:4eadb57ff84c
Date: 2016-11-28 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/4eadb57ff84c/

Log:	ensure that the bio object is also deallocated

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1296,7 +1296,7 @@
         lib.BIO_set_retry_read(bio);
         lib.BIO_set_mem_eof_return(bio, -1);
 
-        self.bio = bio;
+        self.bio = lib.gc(bio, lib.BIO_free)
         self.eof_written = False
 
     @property

From pypy.commits at gmail.com  Mon Nov 28 10:23:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 07:23:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: avoid calling ffi.gc(p,
 ...) where p == ffi.NULL
Message-ID: <583c4bec.0a22c20a.3594d.2c2f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88701:aaca42d1e26e
Date: 2016-11-28 15:43 +0100
http://bitbucket.org/pypy/pypy/changeset/aaca42d1e26e/

Log:	avoid calling ffi.gc(p, ...) where p == ffi.NULL

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -345,7 +345,11 @@
         if ret < 1:
             raise pyssl_error(self, ret)
 
-        self.peer_cert = lib.gc(lib.SSL_get_peer_certificate(ssl), lib.X509_free)
+        peer_cert = lib.SSL_get_peer_certificate(ssl)
+        if peer_cert != ffi.NULL:
+            peer_cert = lib.gc(peer_cert, lib.X509_free)
+        self.peer_cert = peer_cert
+
         #PySSL_END_ALLOW_THREADS
         self.handshake_done = 1
         return None
@@ -730,9 +734,10 @@
         else:
             raise ValueError("invalid protocol version")
 
+        ctx = lib.SSL_CTX_new(method)
+        if ctx == ffi.NULL:
+            raise ssl_error("failed to allocate SSL context")
         self.ctx = ffi.gc(lib.SSL_CTX_new(method), lib.SSL_CTX_free)
-        if self.ctx == ffi.NULL: 
-            raise ssl_error("failed to allocate SSL context")
 
         self._check_hostname = False
 

From pypy.commits at gmail.com  Mon Nov 28 10:23:25 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 07:23:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: it is ffi.gc, not lib.gc,
 use ffi.new("type[]", count) instead of ffi.new("type[%d]"%count),
 special case _str_to_ffi_buffer
Message-ID: <583c4bed.6737c20a.58d18.27fe@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88702:784e2faa7a3d
Date: 2016-11-28 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/784e2faa7a3d/

Log:	it is ffi.gc, not lib.gc, use ffi.new("type[]", count) instead of
	ffi.new("type[%d]"%count), special case _str_to_ffi_buffer

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -203,9 +203,6 @@
 SOCKET_TOO_LARGE_FOR_SELECT = 4
 SOCKET_OPERATION_OK = 5
 
-def _buffer_new(length):
-    return ffi.new("char[%d]"%length)
-
 class _SSLSocket(object):
 
     @staticmethod
@@ -218,7 +215,7 @@
 
         lib.ERR_get_state()
         lib.ERR_clear_error()
-        self.ssl = ssl = lib.gc(lib.SSL_new(ctx), lib.SSL_free)
+        self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 
         self._app_data_handle = ffi.new_handle(self)
         lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
@@ -347,7 +344,7 @@
 
         peer_cert = lib.SSL_get_peer_certificate(ssl)
         if peer_cert != ffi.NULL:
-            peer_cert = lib.gc(peer_cert, lib.X509_free)
+            peer_cert = ffi.gc(peer_cert, lib.X509_free)
         self.peer_cert = peer_cert
 
         #PySSL_END_ALLOW_THREADS
@@ -437,7 +434,7 @@
         sock = self.get_socket_or_connection_gone()
 
         if not buffer_into:
-            dest = _buffer_new(length)
+            dest = ffi.new("char[]", length)
             mem = dest
         else:
             mem = ffi.from_buffer(buffer_into)
@@ -654,7 +651,7 @@
             return count
 
     def tls_unique_cb(self):
-        buf = ffi.new("char[%d]" % SSL_CB_MAXLEN)
+        buf = ffi.new("char[]", SSL_CB_MAXLEN)
 
         if lib.SSL_session_reused(self.ssl) ^ (not self.socket_type):
             # if session is resumed XOR we are the client
@@ -1259,7 +1256,7 @@
         raise ValueError("Unknown object")
     sn = _str_from_buf(lib.OBJ_nid2sn(nid))
     ln = _str_from_buf(lib.OBJ_nid2ln(nid))
-    buf = ffi.new("char[255]")
+    buf = ffi.new("char[]", 255)
     length = lib.OBJ_obj2txt(buf, len(buf), obj, 1)
     if length < 0:
         ssl_error(None)
@@ -1301,7 +1298,7 @@
         lib.BIO_set_retry_read(bio);
         lib.BIO_set_mem_eof_return(bio, -1);
 
-        self.bio = lib.gc(bio, lib.BIO_free)
+        self.bio = ffi.gc(bio, lib.BIO_free)
         self.eof_written = False
 
     @property
@@ -1339,7 +1336,7 @@
         if count < 0 or count > avail:
             count = avail;
 
-        buf = ffi.new("char[%d]" % count)
+        buf = ffi.new("char[]", count)
 
         nbytes = lib.BIO_read(self.bio, buf, count);
         #  There should never be any short reads but check anyway.
@@ -1358,7 +1355,7 @@
 def _RAND_bytes(count, pseudo):
     if count < 0:
         raise ValueError("num must be positive")
-    buf = ffi.new("unsigned char[%d]" % count)
+    buf = ffi.new("unsigned char[]", count)
     if pseudo:
         # note by reaperhulk, RAND_pseudo_bytes is deprecated in 3.6 already,
         # it is totally fine to just call RAND_bytes instead
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -14,14 +14,19 @@
     return ffi.buffer(char_ptr, length)[:]
 
 def _str_to_ffi_buffer(view, zeroterm=False):
+    # XXX incomplete and does not work if e.g. view in (True, 0, 1, ...)
     if zeroterm:
         # only two cases use zeroterm=True, those are rather 'short' strings
+        if isinstance(view, str):
+            return ffi.from_buffer(bytes(view+'\x00', 'utf-8'))
         return ffi.from_buffer(bytes(view)+b'\x00')
 
     try:
         buf = ffi.from_buffer(view)
         return buf
     except TypeError:
+        if isinstance(view, str):
+            return ffi.from_buffer(bytes(view, 'utf-8'))
         return ffi.from_buffer(bytes(view))
 
 def _str_from_buf(buf):

From pypy.commits at gmail.com  Mon Nov 28 10:41:54 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 07:41:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: remove zeroterm parameter of
 _str_to_ffi_buffer
Message-ID: <583c5042.44e61c0a.17aa8.e8fe@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88703:582cca1fe05c
Date: 2016-11-28 16:40 +0100
http://bitbucket.org/pypy/pypy/changeset/582cca1fe05c/

Log:	remove zeroterm parameter of _str_to_ffi_buffer

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -678,6 +678,11 @@
 
 def _fs_decode(name):
     return name.decode(sys.getfilesystemencoding())
+def _fs_converter(name):
+    """ name must not be None """
+    if isinstance(name, str):
+        return name.encode(sys.getfilesystemencoding())
+    return bytes(name)
 
 
 def cipher_to_tuple(cipher):
@@ -1048,7 +1053,7 @@
         ffi.errno = 0
         if filepath is None:
             raise TypeError("filepath must not be None")
-        buf = _str_to_ffi_buffer(filepath, zeroterm=True)
+        buf = _fs_converter(filepath)
         mode = ffi.new("char[]",b"r")
         ffi.errno = 0
         bio = lib.BIO_new_file(buf, mode)
@@ -1096,7 +1101,10 @@
         return _list
 
     def set_ecdh_curve(self, name):
-        buf = _str_to_ffi_buffer(name, zeroterm=True)
+        # needs to be zero terminated
+        if name is None:
+            raise TypeError()
+        buf = _fs_converter(name)
         nid = lib.OBJ_sn2nid(buf)
         if nid == 0:
             raise ValueError("unknown elliptic curve name '%s'" % name)
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -13,14 +13,8 @@
 def _bytes_with_len(char_ptr, length):
     return ffi.buffer(char_ptr, length)[:]
 
-def _str_to_ffi_buffer(view, zeroterm=False):
+def _str_to_ffi_buffer(view):
     # XXX incomplete and does not work if e.g. view in (True, 0, 1, ...)
-    if zeroterm:
-        # only two cases use zeroterm=True, those are rather 'short' strings
-        if isinstance(view, str):
-            return ffi.from_buffer(bytes(view+'\x00', 'utf-8'))
-        return ffi.from_buffer(bytes(view)+b'\x00')
-
     try:
         buf = ffi.from_buffer(view)
         return buf

From pypy.commits at gmail.com  Mon Nov 28 10:52:18 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 07:52:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: revert _str_to_ffi_buffer back to
 parts of it's original impl. str and memoryview are special cased!
Message-ID: <583c52b2.c64bc20a.a1c99.2402@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88704:e44522edcb3a
Date: 2016-11-28 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/e44522edcb3a/

Log:	revert _str_to_ffi_buffer back to parts of it's original impl. str
	and memoryview are special cased!

diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -14,14 +14,15 @@
     return ffi.buffer(char_ptr, length)[:]
 
 def _str_to_ffi_buffer(view):
-    # XXX incomplete and does not work if e.g. view in (True, 0, 1, ...)
-    try:
-        buf = ffi.from_buffer(view)
-        return buf
-    except TypeError:
-        if isinstance(view, str):
-            return ffi.from_buffer(bytes(view, 'utf-8'))
-        return ffi.from_buffer(bytes(view))
+    if isinstance(view, str):
+        return ffi.from_buffer(view.encode())
+    elif isinstance(view, memoryview):
+        # NOTE pypy limitation StringBuffer does not allow
+        # to get a raw address to the string!
+        view = bytes(view)
+    # dont call call ffi.from_buffer(bytes(view)), arguments
+    # like ints/bools should result in a TypeError
+    return ffi.from_buffer(view)
 
 def _str_from_buf(buf):
     return ffi.string(buf).decode('utf-8')

From pypy.commits at gmail.com  Mon Nov 28 10:53:24 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 07:53:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: (arigato,
 plan_rich) merge enhancements to py3.5-ssl
Message-ID: <583c52f4.8f95c20a.2b130.2e09@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r88705:851b07eec964
Date: 2016-11-28 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/851b07eec964/

Log:	(arigato, plan_rich) merge enhancements to py3.5-ssl

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -20,6 +20,8 @@
         SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE,
         pyerr_write_unraisable)
 from _cffi_ssl._stdssl import error
+from select import poll, POLLIN, POLLOUT, select
+from enum import IntEnum as _IntEnum
 
 OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8')
 OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER
@@ -77,7 +79,6 @@
 
 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
 
-from enum import IntEnum as _IntEnum
 _IntEnum._convert('_SSLMethod', __name__,
         lambda name: name.startswith('PROTOCOL_'))
 
@@ -120,7 +121,6 @@
 def _Cryptography_pem_password_cb(buf, size, rwflag, userdata):
     pw_info = ffi.from_handle(userdata)
 
-    # TODO PySSL_END_ALLOW_THREADS_S(pw_info->thread_state);
     password = pw_info.password
 
     if pw_info.callable:
@@ -140,7 +140,6 @@
         pw_info.operationerror = ValueError("password cannot be longer than %d bytes" % size)
         return 0
 
-    #PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state);
     ffi.memmove(buf, password, len(password))
     return len(password)
 
@@ -150,7 +149,12 @@
 else:
     Cryptography_pem_password_cb = ffi.callback("int(char*,int,int,void*)")(_Cryptography_pem_password_cb)
 
-from select import poll, POLLIN, POLLOUT, select
+if hasattr(time, 'monotonic'):
+    def _monotonic_clock():
+        return time.monotonic()
+else:
+    def _monotonic_clock():
+        return time.clock_gettime(time.CLOCK_MONOTONIC)
 
 HAVE_POLL = True
 
@@ -177,10 +181,9 @@
     if HAVE_POLL:
         p.register(sock.fileno(), POLLOUT | POLLIN)
 
-        #PySSL_BEGIN_ALLOW_THREADS
         rc = len(p.poll(timeout * 1000.0))
-        #PySSL_END_ALLOW_THREADS
     else:
+        # currently disabled, see HAVE_POLL
         fd = sock.fileno()
         #if (!_PyIsSelectable_fd(s->sock_fd))
         #    return SOCKET_TOO_LARGE_FOR_SELECT;
@@ -200,9 +203,6 @@
 SOCKET_TOO_LARGE_FOR_SELECT = 4
 SOCKET_OPERATION_OK = 5
 
-def _buffer_new(length):
-    return ffi.new("char[%d]"%length)
-
 class _SSLSocket(object):
 
     @staticmethod
@@ -215,9 +215,9 @@
 
         lib.ERR_get_state()
         lib.ERR_clear_error()
-        self.ssl = ssl = lib.SSL_new(ctx)
+        self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 
-        self._app_data_handle = ffi.new_handle(self) # TODO release later
+        self._app_data_handle = ffi.new_handle(self)
         lib.SSL_set_app_data(ssl, ffi.cast("char*", self._app_data_handle))
         if sock:
             lib.SSL_set_fd(ssl, sock.fileno())
@@ -247,13 +247,11 @@
             lib.BIO_set_nbio(lib.SSL_get_rbio(ssl), 1)
             lib.BIO_set_nbio(lib.SSL_get_wbio(ssl), 1)
 
-        #PySSL_BEGIN_ALLOW_THREADS
         if socket_type == SSL_CLIENT:
             lib.SSL_set_connect_state(ssl)
         else:
             lib.SSL_set_accept_state(ssl)
         self.socket_type = socket_type
-        #PySSL_END_ALLOW_THREADS
 
         if sock:
             self.socket = weakref.ref(sock)
@@ -309,8 +307,7 @@
         has_timeout = timeout > 0
         deadline = -1
         if has_timeout:
-            # REVIEW, cpython uses a monotonic clock here
-            deadline = time.time() + timeout;
+            deadline = _monotonic_clock() + timeout;
         # Actually negotiate SSL connection
         # XXX If SSL_do_handshake() returns 0, it's also a failure.
         while True:
@@ -323,7 +320,7 @@
 
             if has_timeout:
                 # REIVIEW monotonic clock?
-                timeout = deadline - time.time()
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -345,10 +342,11 @@
         if ret < 1:
             raise pyssl_error(self, ret)
 
-        if self.peer_cert != ffi.NULL:
-            lib.X509_free(self.peer_cert)
-        #PySSL_BEGIN_ALLOW_THREADS
-        self.peer_cert = lib.SSL_get_peer_certificate(ssl)
+        peer_cert = lib.SSL_get_peer_certificate(ssl)
+        if peer_cert != ffi.NULL:
+            peer_cert = ffi.gc(peer_cert, lib.X509_free)
+        self.peer_cert = peer_cert
+
         #PySSL_END_ALLOW_THREADS
         self.handshake_done = 1
         return None
@@ -387,8 +385,7 @@
 
         has_timeout = timeout > 0
         if has_timeout:
-            # TODO monotonic clock?
-            deadline = time.time() + timeout
+            deadline = _monotonic_clock() + timeout
 
         sockstate = _ssl_select(sock, 1, timeout)
         if sockstate == SOCKET_HAS_TIMED_OUT:
@@ -399,16 +396,13 @@
             raise ssl_error("Underlying socket too large for select().")
 
         while True:
-            #PySSL_START_ALLOW_THREADS
             length = lib.SSL_write(self.ssl, b, len(b))
             err = lib.SSL_get_error(self.ssl, length)
-            #PySSL_END_ALLOW_THREADS
 
             check_signals()
 
             if has_timeout:
-                # TODO monotonic clock
-                timeout = deadline - time.time()
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -440,7 +434,7 @@
         sock = self.get_socket_or_connection_gone()
 
         if not buffer_into:
-            dest = _buffer_new(length)
+            dest = ffi.new("char[]", length)
             mem = dest
         else:
             mem = ffi.from_buffer(buffer_into)
@@ -459,20 +453,17 @@
         timeout = _socket_timeout(sock)
         has_timeout = timeout > 0
         if has_timeout:
-            # TODO monotonic clock?
-            deadline = time.time() + timeout
+            deadline = _monotonic_clock() + timeout
 
         shutdown = False
         while True:
-            #PySSL_BEGIN_ALLOW_THREADS
             count = lib.SSL_read(self.ssl, mem, length);
             err = lib.SSL_get_error(self.ssl, count);
-            #PySSL_END_ALLOW_THREADS
 
             check_signals()
 
             if has_timeout:
-                timeout = deadline - time.time() # TODO ? _PyTime_GetMonotonicClock();
+                timeout = deadline - _monotonic_clock()
 
             if err == SSL_ERROR_WANT_READ:
                 sockstate = _ssl_select(sock, 0, timeout)
@@ -591,13 +582,11 @@
 
         has_timeout = (timeout > 0);
         if has_timeout:
-            # TODO monotonic clock
-            deadline = time.time() + timeout;
+            deadline = _monotonic_clock() + timeout;
 
         zeros = 0
 
         while True:
-            # TODO PySSL_BEGIN_ALLOW_THREADS
             # Disable read-ahead so that unwrap can work correctly.
             # Otherwise OpenSSL might read in too much data,
             # eating clear text data that happens to be
@@ -609,7 +598,6 @@
             if self.shutdown_seen_zero:
                 lib.SSL_set_read_ahead(self.ssl, 0)
             err = lib.SSL_shutdown(self.ssl)
-            # TODO PySSL_END_ALLOW_THREADS
 
             # If err == 1, a secure shutdown with SSL_shutdown() is complete
             if err > 0:
@@ -626,8 +614,7 @@
                 continue
 
             if has_timeout:
-                # TODO monotonic clock
-                timeout = deadline - time.time() #_PyTime_GetMonotonicClock();
+                timeout = deadline - _monotonic_clock()
 
             # Possibly retry shutdown until timeout or failure
             ssl_err = lib.SSL_get_error(self.ssl, err)
@@ -657,16 +644,14 @@
             return None
 
     def pending(self):
-        # TODO PySSL_BEGIN_ALLOW_THREADS
         count = lib.SSL_pending(self.ssl)
-        # TODO PySSL_END_ALLOW_THREADS
         if count < 0:
             raise pyssl_error(self, count)
         else:
             return count
 
     def tls_unique_cb(self):
-        buf = ffi.new("char[%d]" % SSL_CB_MAXLEN)
+        buf = ffi.new("char[]", SSL_CB_MAXLEN)
 
         if lib.SSL_session_reused(self.ssl) ^ (not self.socket_type):
             # if session is resumed XOR we are the client
@@ -692,8 +677,12 @@
 
 
 def _fs_decode(name):
-    # TODO return PyUnicode_DecodeFSDefault(short_name);
-    return name.decode('utf-8')
+    return name.decode(sys.getfilesystemencoding())
+def _fs_converter(name):
+    """ name must not be None """
+    if isinstance(name, str):
+        return name.encode(sys.getfilesystemencoding())
+    return bytes(name)
 
 
 def cipher_to_tuple(cipher):
@@ -747,9 +736,10 @@
         else:
             raise ValueError("invalid protocol version")
 
-        self.ctx = lib.SSL_CTX_new(method)
-        if self.ctx == ffi.NULL: 
+        ctx = lib.SSL_CTX_new(method)
+        if ctx == ffi.NULL:
             raise ssl_error("failed to allocate SSL context")
+        self.ctx = ffi.gc(lib.SSL_CTX_new(method), lib.SSL_CTX_free)
 
         self._check_hostname = False
 
@@ -1049,15 +1039,6 @@
         return {'x509': x509, 'x509_ca': x509_ca, 'crl': crl}
 
 
-    def __del__(self):
-        # REVIEW, is this done properly? In RPython the a finalizer
-        # function is added
-        ctx = self.ctx
-        if ctx:
-            self.ctx = None
-            lib.SSL_CTX_free(ctx)
-
-
     def session_stats(self):
         stats = {}
         for name, ssl_func in SSL_CTX_STATS:
@@ -1072,7 +1053,7 @@
         ffi.errno = 0
         if filepath is None:
             raise TypeError("filepath must not be None")
-        buf = _str_to_ffi_buffer(filepath, zeroterm=True)
+        buf = _fs_converter(filepath)
         mode = ffi.new("char[]",b"r")
         ffi.errno = 0
         bio = lib.BIO_new_file(buf, mode)
@@ -1120,7 +1101,10 @@
         return _list
 
     def set_ecdh_curve(self, name):
-        buf = _str_to_ffi_buffer(name, zeroterm=True)
+        # needs to be zero terminated
+        if name is None:
+            raise TypeError()
+        buf = _fs_converter(name)
         nid = lib.OBJ_sn2nid(buf)
         if nid == 0:
             raise ValueError("unknown elliptic curve name '%s'" % name)
@@ -1273,7 +1257,6 @@
         self.ctx = ctx
 
 SERVERNAME_CALLBACKS = weakref.WeakValueDictionary()
-TEST = None
 
 def _asn1obj2py(obj):
     nid = lib.OBJ_obj2nid(obj)
@@ -1281,7 +1264,7 @@
         raise ValueError("Unknown object")
     sn = _str_from_buf(lib.OBJ_nid2sn(nid))
     ln = _str_from_buf(lib.OBJ_nid2ln(nid))
-    buf = ffi.new("char[255]")
+    buf = ffi.new("char[]", 255)
     length = lib.OBJ_obj2txt(buf, len(buf), obj, 1)
     if length < 0:
         ssl_error(None)
@@ -1323,7 +1306,7 @@
         lib.BIO_set_retry_read(bio);
         lib.BIO_set_mem_eof_return(bio, -1);
 
-        self.bio = bio;
+        self.bio = ffi.gc(bio, lib.BIO_free)
         self.eof_written = False
 
     @property
@@ -1361,7 +1344,7 @@
         if count < 0 or count > avail:
             count = avail;
 
-        buf = ffi.new("char[%d]" % count)
+        buf = ffi.new("char[]", count)
 
         nbytes = lib.BIO_read(self.bio, buf, count);
         #  There should never be any short reads but check anyway.
@@ -1376,12 +1359,11 @@
 
 
 RAND_status = lib.RAND_status
-RAND_add = lib.RAND_add
 
 def _RAND_bytes(count, pseudo):
     if count < 0:
         raise ValueError("num must be positive")
-    buf = ffi.new("unsigned char[%d]" % count)
+    buf = ffi.new("unsigned char[]", count)
     if pseudo:
         # note by reaperhulk, RAND_pseudo_bytes is deprecated in 3.6 already,
         # it is totally fine to just call RAND_bytes instead
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -278,7 +278,6 @@
         lib.BIO_free(cert)
         raise ssl_error("Can't malloc memory to read file")
 
-    # REVIEW how to encode this properly?
     epath = path.encode()
     if lib.BIO_read_filename(cert, epath) <= 0:
         lib.BIO_free(cert)
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -13,39 +13,22 @@
 def _bytes_with_len(char_ptr, length):
     return ffi.buffer(char_ptr, length)[:]
 
-def _str_to_ffi_buffer(view, zeroterm=False):
-    # REVIEW unsure how to solve this. might be easy:
-    # str does not support buffer protocol.
-    # I think a user should really encode the string before it is 
-    # passed here!
+def _str_to_ffi_buffer(view):
     if isinstance(view, str):
-        enc = view.encode()
-        if zeroterm:
-            return ffi.from_buffer(enc + b'\x00')
-        else:
-            return ffi.from_buffer(enc)
-    else:
-        if isinstance(view, memoryview):
-            # TODO pypy limitation StringBuffer does not allow
-            # to get a raw address to the string!
-            view = bytes(view)
-        if zeroterm:
-            return ffi.from_buffer(view + b'\x00')
-        else:
-            return ffi.from_buffer(view)
+        return ffi.from_buffer(view.encode())
+    elif isinstance(view, memoryview):
+        # NOTE pypy limitation StringBuffer does not allow
+        # to get a raw address to the string!
+        view = bytes(view)
+    # dont call call ffi.from_buffer(bytes(view)), arguments
+    # like ints/bools should result in a TypeError
+    return ffi.from_buffer(view)
 
 def _str_from_buf(buf):
     return ffi.string(buf).decode('utf-8')
 
 def _cstr_decode_fs(buf):
-#define CONVERT(info, target) { \
-#        const char *tmp = (info); \
-#        target = NULL; \
-#        if (!tmp) { Py_INCREF(Py_None); target = Py_None; } \
-#        else if ((target = PyUnicode_DecodeFSDefault(tmp)) == NULL) { \
-#            target = PyBytes_FromString(tmp); } \
-#        if (!target) goto error; \
-#    }
-    # REVIEW
+    if buf == ffi.NULL:
+        return None
     return ffi.string(buf).decode(sys.getfilesystemencoding())
 
diff --git a/lib_pypy/ssl.py b/lib_pypy/ssl.py
deleted file mode 100644
--- a/lib_pypy/ssl.py
+++ /dev/null
@@ -1,1148 +0,0 @@
-# Wrapper module for _ssl, providing some additional facilities
-# implemented in Python.  Written by Bill Janssen.
-
-"""This module provides some more Pythonic support for SSL.
-
-Object types:
-
-  SSLSocket -- subtype of socket.socket which does SSL over the socket
-
-Exceptions:
-
-  SSLError -- exception raised for I/O errors
-
-Functions:
-
-  cert_time_to_seconds -- convert time string used for certificate
-                          notBefore and notAfter functions to integer
-                          seconds past the Epoch (the time values
-                          returned from time.time())
-
-  fetch_server_certificate (HOST, PORT) -- fetch the certificate provided
-                          by the server running on HOST at port PORT.  No
-                          validation of the certificate is performed.
-
-Integer constants:
-
-SSL_ERROR_ZERO_RETURN
-SSL_ERROR_WANT_READ
-SSL_ERROR_WANT_WRITE
-SSL_ERROR_WANT_X509_LOOKUP
-SSL_ERROR_SYSCALL
-SSL_ERROR_SSL
-SSL_ERROR_WANT_CONNECT
-
-SSL_ERROR_EOF
-SSL_ERROR_INVALID_ERROR_CODE
-
-The following group define certificate requirements that one side is
-allowing/requiring from the other side:
-
-CERT_NONE - no certificates from the other side are required (or will
-            be looked at if provided)
-CERT_OPTIONAL - certificates are not required, but if provided will be
-                validated, and if validation fails, the connection will
-                also fail
-CERT_REQUIRED - certificates are required, and will be validated, and
-                if validation fails, the connection will also fail
-
-The following constants identify various SSL protocol variants:
-
-PROTOCOL_SSLv2
-PROTOCOL_SSLv3
-PROTOCOL_SSLv23
-PROTOCOL_TLSv1
-PROTOCOL_TLSv1_1
-PROTOCOL_TLSv1_2
-
-The following constants identify various SSL alert message descriptions as per
-http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6
-
-ALERT_DESCRIPTION_CLOSE_NOTIFY
-ALERT_DESCRIPTION_UNEXPECTED_MESSAGE
-ALERT_DESCRIPTION_BAD_RECORD_MAC
-ALERT_DESCRIPTION_RECORD_OVERFLOW
-ALERT_DESCRIPTION_DECOMPRESSION_FAILURE
-ALERT_DESCRIPTION_HANDSHAKE_FAILURE
-ALERT_DESCRIPTION_BAD_CERTIFICATE
-ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE
-ALERT_DESCRIPTION_CERTIFICATE_REVOKED
-ALERT_DESCRIPTION_CERTIFICATE_EXPIRED
-ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN
-ALERT_DESCRIPTION_ILLEGAL_PARAMETER
-ALERT_DESCRIPTION_UNKNOWN_CA
-ALERT_DESCRIPTION_ACCESS_DENIED
-ALERT_DESCRIPTION_DECODE_ERROR
-ALERT_DESCRIPTION_DECRYPT_ERROR
-ALERT_DESCRIPTION_PROTOCOL_VERSION
-ALERT_DESCRIPTION_INSUFFICIENT_SECURITY
-ALERT_DESCRIPTION_INTERNAL_ERROR
-ALERT_DESCRIPTION_USER_CANCELLED
-ALERT_DESCRIPTION_NO_RENEGOTIATION
-ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION
-ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
-ALERT_DESCRIPTION_UNRECOGNIZED_NAME
-ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE
-ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE
-ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY
-"""
-
-import ipaddress
-import textwrap
-import re
-import sys
-import os
-from collections import namedtuple
-from enum import Enum as _Enum, IntEnum as _IntEnum
-
-import _ssl             # if we can't import it, let the error propagate
-
-from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
-from _ssl import _SSLContext, MemoryBIO
-from _ssl import (
-    SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError,
-    SSLSyscallError, SSLEOFError,
-    )
-from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj
-from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes
-try:
-    from _ssl import RAND_egd
-except ImportError:
-    # LibreSSL does not provide RAND_egd
-    pass
-
-def _import_symbols(prefix):
-    for n in dir(_ssl):
-        if n.startswith(prefix):
-            globals()[n] = getattr(_ssl, n)
-
-_import_symbols('OP_')
-_import_symbols('ALERT_DESCRIPTION_')
-_import_symbols('SSL_ERROR_')
-_import_symbols('VERIFY_')
-
-from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN
-
-from _ssl import _OPENSSL_API_VERSION
-
-_IntEnum._convert(
-        '_SSLMethod', __name__,
-        lambda name: name.startswith('PROTOCOL_'),
-        source=_ssl)
-
-_PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()}
-
-try:
-    _SSLv2_IF_EXISTS = PROTOCOL_SSLv2
-except NameError:
-    _SSLv2_IF_EXISTS = None
-
-if sys.platform == "win32":
-    from _ssl import enum_certificates, enum_crls
-
-from socket import socket, AF_INET, SOCK_STREAM, create_connection
-from socket import SOL_SOCKET, SO_TYPE
-import base64        # for DER-to-PEM translation
-import errno
-import warnings
-
-
-socket_error = OSError  # keep that public name in module namespace
-
-if _ssl.HAS_TLS_UNIQUE:
-    CHANNEL_BINDING_TYPES = ['tls-unique']
-else:
-    CHANNEL_BINDING_TYPES = []
-
-# Disable weak or insecure ciphers by default
-# (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL')
-# Enable a better set of ciphers by default
-# This list has been explicitly chosen to:
-#   * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
-#   * Prefer ECDHE over DHE for better performance
-#   * Prefer any AES-GCM over any AES-CBC for better performance and security
-#   * Then Use HIGH cipher suites as a fallback
-#   * Then Use 3DES as fallback which is secure but slow
-#   * Disable NULL authentication, NULL encryption, and MD5 MACs for security
-#     reasons
-_DEFAULT_CIPHERS = (
-    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
-    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
-    '!eNULL:!MD5'
-)
-
-# Restricted and more secure ciphers for the server side
-# This list has been explicitly chosen to:
-#   * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
-#   * Prefer ECDHE over DHE for better performance
-#   * Prefer any AES-GCM over any AES-CBC for better performance and security
-#   * Then Use HIGH cipher suites as a fallback
-#   * Then Use 3DES as fallback which is secure but slow
-#   * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, and RC4 for
-#     security reasons
-_RESTRICTED_SERVER_CIPHERS = (
-    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
-    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
-    '!eNULL:!MD5:!DSS:!RC4'
-)
-
-
-class CertificateError(ValueError):
-    pass
-
-
-def _dnsname_match(dn, hostname, max_wildcards=1):
-    """Matching according to RFC 6125, section 6.4.3
-
-    http://tools.ietf.org/html/rfc6125#section-6.4.3
-    """
-    pats = []
-    if not dn:
-        return False
-
-    leftmost, *remainder = dn.split(r'.')
-
-    wildcards = leftmost.count('*')
-    if wildcards > max_wildcards:
-        # Issue #17980: avoid denials of service by refusing more
-        # than one wildcard per fragment.  A survey of established
-        # policy among SSL implementations showed it to be a
-        # reasonable choice.
-        raise CertificateError(
-            "too many wildcards in certificate DNS name: " + repr(dn))
-
-    # speed up common case w/o wildcards
-    if not wildcards:
-        return dn.lower() == hostname.lower()
-
-    # RFC 6125, section 6.4.3, subitem 1.
-    # The client SHOULD NOT attempt to match a presented identifier in which
-    # the wildcard character comprises a label other than the left-most label.
-    if leftmost == '*':
-        # When '*' is a fragment by itself, it matches a non-empty dotless
-        # fragment.
-        pats.append('[^.]+')
-    elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
-        # RFC 6125, section 6.4.3, subitem 3.
-        # The client SHOULD NOT attempt to match a presented identifier
-        # where the wildcard character is embedded within an A-label or
-        # U-label of an internationalized domain name.
-        pats.append(re.escape(leftmost))
-    else:
-        # Otherwise, '*' matches any dotless string, e.g. www*
-        pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
-
-    # add the remaining fragments, ignore any wildcards
-    for frag in remainder:
-        pats.append(re.escape(frag))
-
-    pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
-    return pat.match(hostname)
-
-
-def _ipaddress_match(ipname, host_ip):
-    """Exact matching of IP addresses.
-
-    RFC 6125 explicitly doesn't define an algorithm for this
-    (section 1.7.2 - "Out of Scope").
-    """
-    # OpenSSL may add a trailing newline to a subjectAltName's IP address
-    ip = ipaddress.ip_address(ipname.rstrip())
-    return ip == host_ip
-
-
-def match_hostname(cert, hostname):
-    """Verify that *cert* (in decoded format as returned by
-    SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
-    rules are followed, but IP addresses are not accepted for *hostname*.
-
-    CertificateError is raised on failure. On success, the function
-    returns nothing.
-    """
-    if not cert:
-        raise ValueError("empty or no certificate, match_hostname needs a "
-                         "SSL socket or SSL context with either "
-                         "CERT_OPTIONAL or CERT_REQUIRED")
-    try:
-        host_ip = ipaddress.ip_address(hostname)
-    except ValueError:
-        # Not an IP address (common case)
-        host_ip = None
-    dnsnames = []
-    san = cert.get('subjectAltName', ())
-    for key, value in san:
-        if key == 'DNS':
-            if host_ip is None and _dnsname_match(value, hostname):
-                return
-            dnsnames.append(value)
-        elif key == 'IP Address':
-            if host_ip is not None and _ipaddress_match(value, host_ip):
-                return
-            dnsnames.append(value)
-    if not dnsnames:
-        # The subject is only checked when there is no dNSName entry
-        # in subjectAltName
-        for sub in cert.get('subject', ()):
-            for key, value in sub:
-                # XXX according to RFC 2818, the most specific Common Name
-                # must be used.
-                if key == 'commonName':
-                    if _dnsname_match(value, hostname):
-                        return
-                    dnsnames.append(value)
-    if len(dnsnames) > 1:
-        raise CertificateError("hostname %r "
-            "doesn't match either of %s"
-            % (hostname, ', '.join(map(repr, dnsnames))))
-    elif len(dnsnames) == 1:
-        raise CertificateError("hostname %r "
-            "doesn't match %r"
-            % (hostname, dnsnames[0]))
-    else:
-        raise CertificateError("no appropriate commonName or "
-            "subjectAltName fields were found")
-
-
-DefaultVerifyPaths = namedtuple("DefaultVerifyPaths",
-    "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env "
-    "openssl_capath")
-
-def get_default_verify_paths():
-    """Return paths to default cafile and capath.
-    """
-    parts = _ssl.get_default_verify_paths()
-
-    # environment vars shadow paths
-    cafile = os.environ.get(parts[0], parts[1])
-    capath = os.environ.get(parts[2], parts[3])
-
-    return DefaultVerifyPaths(cafile if os.path.isfile(cafile) else None,
-                              capath if os.path.isdir(capath) else None,
-                              *parts)
-
-
-class _ASN1Object(namedtuple("_ASN1Object", "nid shortname longname oid")):
-    """ASN.1 object identifier lookup
-    """
-    __slots__ = ()
-
-    def __new__(cls, oid):
-        return super().__new__(cls, *_txt2obj(oid, name=False))
-
-    @classmethod
-    def fromnid(cls, nid):
-        """Create _ASN1Object from OpenSSL numeric ID
-        """
-        return super().__new__(cls, *_nid2obj(nid))
-
-    @classmethod
-    def fromname(cls, name):
-        """Create _ASN1Object from short name, long name or OID
-        """
-        return super().__new__(cls, *_txt2obj(name, name=True))
-
-
-class Purpose(_ASN1Object, _Enum):
-    """SSLContext purpose flags with X509v3 Extended Key Usage objects
-    """
-    SERVER_AUTH = '1.3.6.1.5.5.7.3.1'
-    CLIENT_AUTH = '1.3.6.1.5.5.7.3.2'
-
-
-class SSLContext(_SSLContext):
-    """An SSLContext holds various SSL-related configuration options and
-    data, such as certificates and possibly a private key."""
-
-    __slots__ = ('protocol', '__weakref__')
-    _windows_cert_stores = ("CA", "ROOT")
-
-    def __new__(cls, protocol, *args, **kwargs):
-        self = _SSLContext.__new__(cls, protocol)
-        if protocol != _SSLv2_IF_EXISTS:
-            self.set_ciphers(_DEFAULT_CIPHERS)
-        return self
-
-    def __init__(self, protocol):
-        self.protocol = protocol
-
-    def wrap_socket(self, sock, server_side=False,
-                    do_handshake_on_connect=True,
-                    suppress_ragged_eofs=True,
-                    server_hostname=None):
-        return SSLSocket(sock=sock, server_side=server_side,
-                         do_handshake_on_connect=do_handshake_on_connect,
-                         suppress_ragged_eofs=suppress_ragged_eofs,
-                         server_hostname=server_hostname,
-                         _context=self)
-
-    def wrap_bio(self, incoming, outgoing, server_side=False,
-                 server_hostname=None):
-        sslobj = self._wrap_bio(incoming, outgoing, server_side=server_side,
-                                server_hostname=server_hostname)
-        return SSLObject(sslobj)
-
-    def set_npn_protocols(self, npn_protocols):
-        protos = bytearray()
-        for protocol in npn_protocols:
-            b = bytes(protocol, 'ascii')
-            if len(b) == 0 or len(b) > 255:
-                raise SSLError('NPN protocols must be 1 to 255 in length')
-            protos.append(len(b))
-            protos.extend(b)
-
-        self._set_npn_protocols(protos)
-
-    def set_alpn_protocols(self, alpn_protocols):
-        protos = bytearray()
-        for protocol in alpn_protocols:
-            b = bytes(protocol, 'ascii')
-            if len(b) == 0 or len(b) > 255:
-                raise SSLError('ALPN protocols must be 1 to 255 in length')
-            protos.append(len(b))
-            protos.extend(b)
-
-        self._set_alpn_protocols(protos)
-
-    def _load_windows_store_certs(self, storename, purpose):
-        certs = bytearray()
-        try:
-            for cert, encoding, trust in enum_certificates(storename):
-                # CA certs are never PKCS#7 encoded
-                if encoding == "x509_asn":
-                    if trust is True or purpose.oid in trust:
-                        certs.extend(cert)
-        except PermissionError:
-            warnings.warn("unable to enumerate Windows certificate store")
-        if certs:
-            self.load_verify_locations(cadata=certs)
-        return certs
-
-    def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
-        if not isinstance(purpose, _ASN1Object):
-            raise TypeError(purpose)
-        if sys.platform == "win32":
-            for storename in self._windows_cert_stores:
-                self._load_windows_store_certs(storename, purpose)
-        self.set_default_verify_paths()
-
-
-def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None,
-                           capath=None, cadata=None):
-    """Create a SSLContext object with default settings.
-
-    NOTE: The protocol and settings may change anytime without prior
-          deprecation. The values represent a fair balance between maximum
-          compatibility and security.
-    """
-    if not isinstance(purpose, _ASN1Object):
-        raise TypeError(purpose)
-
-    context = SSLContext(PROTOCOL_SSLv23)
-
-    # SSLv2 considered harmful.
-    context.options |= OP_NO_SSLv2
-
-    # SSLv3 has problematic security and is only required for really old
-    # clients such as IE6 on Windows XP
-    context.options |= OP_NO_SSLv3
-
-    # disable compression to prevent CRIME attacks (OpenSSL 1.0+)
-    context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
-
-    if purpose == Purpose.SERVER_AUTH:
-        # verify certs and host name in client mode
-        context.verify_mode = CERT_REQUIRED
-        context.check_hostname = True
-    elif purpose == Purpose.CLIENT_AUTH:
-        # Prefer the server's ciphers by default so that we get stronger
-        # encryption
-        context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0)
-
-        # Use single use keys in order to improve forward secrecy
-        context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0)
-        context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0)
-
-        # disallow ciphers with known vulnerabilities
-        context.set_ciphers(_RESTRICTED_SERVER_CIPHERS)
-
-    if cafile or capath or cadata:
-        context.load_verify_locations(cafile, capath, cadata)
-    elif context.verify_mode != CERT_NONE:
-        # no explicit cafile, capath or cadata but the verify mode is
-        # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
-        # root CA certificates for the given purpose. This may fail silently.
-        context.load_default_certs(purpose)
-    return context
-
-def _create_unverified_context(protocol=PROTOCOL_SSLv23, *, cert_reqs=None,
-                           check_hostname=False, purpose=Purpose.SERVER_AUTH,
-                           certfile=None, keyfile=None,
-                           cafile=None, capath=None, cadata=None):
-    """Create a SSLContext object for Python stdlib modules
-
-    All Python stdlib modules shall use this function to create SSLContext
-    objects in order to keep common settings in one place. The configuration
-    is less restrict than create_default_context()'s to increase backward
-    compatibility.
-    """
-    if not isinstance(purpose, _ASN1Object):
-        raise TypeError(purpose)
-
-    context = SSLContext(protocol)
-    # SSLv2 considered harmful.
-    context.options |= OP_NO_SSLv2
-    # SSLv3 has problematic security and is only required for really old
-    # clients such as IE6 on Windows XP
-    context.options |= OP_NO_SSLv3
-
-    if cert_reqs is not None:
-        context.verify_mode = cert_reqs
-    context.check_hostname = check_hostname
-
-    if keyfile and not certfile:
-        raise ValueError("certfile must be specified")
-    if certfile or keyfile:
-        context.load_cert_chain(certfile, keyfile)
-
-    # load CA root certs
-    if cafile or capath or cadata:
-        context.load_verify_locations(cafile, capath, cadata)
-    elif context.verify_mode != CERT_NONE:
-        # no explicit cafile, capath or cadata but the verify mode is
-        # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
-        # root CA certificates for the given purpose. This may fail silently.
-        context.load_default_certs(purpose)
-
-    return context
-
-# Used by http.client if no context is explicitly passed.
-_create_default_https_context = create_default_context
-
-
-# Backwards compatibility alias, even though it's not a public name.
-_create_stdlib_context = _create_unverified_context
-
-
-class SSLObject:
-    """This class implements an interface on top of a low-level SSL object as
-    implemented by OpenSSL. This object captures the state of an SSL connection
-    but does not provide any network IO itself. IO needs to be performed
-    through separate "BIO" objects which are OpenSSL's IO abstraction layer.
-
-    This class does not have a public constructor. Instances are returned by
-    ``SSLContext.wrap_bio``. This class is typically used by framework authors
-    that want to implement asynchronous IO for SSL through memory buffers.
-
-    When compared to ``SSLSocket``, this object lacks the following features:
-
-     * Any form of network IO incluging methods such as ``recv`` and ``send``.
-     * The ``do_handshake_on_connect`` and ``suppress_ragged_eofs`` machinery.
-    """
-
-    def __init__(self, sslobj, owner=None):
-        self._sslobj = sslobj
-        # Note: _sslobj takes a weak reference to owner
-        self._sslobj.owner = owner or self
-
-    @property
-    def context(self):
-        """The SSLContext that is currently in use."""
-        return self._sslobj.context
-
-    @context.setter
-    def context(self, ctx):
-        self._sslobj.context = ctx
-
-    @property
-    def server_side(self):
-        """Whether this is a server-side socket."""
-        return self._sslobj.server_side
-
-    @property
-    def server_hostname(self):
-        """The currently set server hostname (for SNI), or ``None`` if no
-        server hostame is set."""
-        return self._sslobj.server_hostname
-
-    def read(self, len=1024, buffer=None):
-        """Read up to 'len' bytes from the SSL object and return them.
-
-        If 'buffer' is provided, read into this buffer and return the number of
-        bytes read.
-        """
-        if buffer is not None:
-            v = self._sslobj.read(len, buffer)
-        else:
-            v = self._sslobj.read(len)
-        return v
-
-    def write(self, data):
-        """Write 'data' to the SSL object and return the number of bytes
-        written.
-
-        The 'data' argument must support the buffer interface.
-        """
-        return self._sslobj.write(data)
-
-    def getpeercert(self, binary_form=False):
-        """Returns a formatted version of the data in the certificate provided
-        by the other end of the SSL channel.
-
-        Return None if no certificate was provided, {} if a certificate was
-        provided, but not validated.
-        """
-        return self._sslobj.peer_certificate(binary_form)
-
-    def selected_npn_protocol(self):
-        """Return the currently selected NPN protocol as a string, or ``None``
-        if a next protocol was not negotiated or if NPN is not supported by one
-        of the peers."""
-        if _ssl.HAS_NPN:
-            return self._sslobj.selected_npn_protocol()
-
-    def selected_alpn_protocol(self):
-        """Return the currently selected ALPN protocol as a string, or ``None``
-        if a next protocol was not negotiated or if ALPN is not supported by one
-        of the peers."""
-        if _ssl.HAS_ALPN:
-            return self._sslobj.selected_alpn_protocol()
-
-    def cipher(self):
-        """Return the currently selected cipher as a 3-tuple ``(name,
-        ssl_version, secret_bits)``."""
-        return self._sslobj.cipher()
-
-    def shared_ciphers(self):
-        """Return a list of ciphers shared by the client during the handshake or
-        None if this is not a valid server connection.
-        """
-        return self._sslobj.shared_ciphers()
-
-    def compression(self):
-        """Return the current compression algorithm in use, or ``None`` if
-        compression was not negotiated or not supported by one of the peers."""
-        return self._sslobj.compression()
-
-    def pending(self):
-        """Return the number of bytes that can be read immediately."""
-        return self._sslobj.pending()
-
-    def do_handshake(self):
-        """Start the SSL/TLS handshake."""
-        self._sslobj.do_handshake()
-        if self.context.check_hostname:
-            if not self.server_hostname:
-                raise ValueError("check_hostname needs server_hostname "
-                                 "argument")
-            match_hostname(self.getpeercert(), self.server_hostname)
-
-    def unwrap(self):
-        """Start the SSL shutdown handshake."""
-        return self._sslobj.shutdown()
-
-    def get_channel_binding(self, cb_type="tls-unique"):
-        """Get channel binding data for current connection.  Raise ValueError
-        if the requested `cb_type` is not supported.  Return bytes of the data
-        or None if the data is not available (e.g. before the handshake)."""
-        if cb_type not in CHANNEL_BINDING_TYPES:
-            raise ValueError("Unsupported channel binding type")
-        if cb_type != "tls-unique":
-            raise NotImplementedError(
-                            "{0} channel binding type not implemented"
-                            .format(cb_type))
-        return self._sslobj.tls_unique_cb()
-
-    def version(self):
-        """Return a string identifying the protocol version used by the
-        current SSL channel. """
-        return self._sslobj.version()
-
-
-class SSLSocket(socket):
-    """This class implements a subtype of socket.socket that wraps
-    the underlying OS socket in an SSL context when necessary, and
-    provides read and write methods over that channel."""
-
-    def __init__(self, sock=None, keyfile=None, certfile=None,
-                 server_side=False, cert_reqs=CERT_NONE,
-                 ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-                 do_handshake_on_connect=True,
-                 family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None,
-                 suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
-                 server_hostname=None,
-                 _context=None):
-
-        if _context:
-            self._context = _context
-        else:
-            if server_side and not certfile:
-                raise ValueError("certfile must be specified for server-side "
-                                 "operations")
-            if keyfile and not certfile:
-                raise ValueError("certfile must be specified")
-            if certfile and not keyfile:
-                keyfile = certfile
-            self._context = SSLContext(ssl_version)
-            self._context.verify_mode = cert_reqs
-            if ca_certs:
-                self._context.load_verify_locations(ca_certs)
-            if certfile:
-                self._context.load_cert_chain(certfile, keyfile)
-            if npn_protocols:
-                self._context.set_npn_protocols(npn_protocols)
-            if ciphers:
-                self._context.set_ciphers(ciphers)
-            self.keyfile = keyfile
-            self.certfile = certfile
-            self.cert_reqs = cert_reqs
-            self.ssl_version = ssl_version
-            self.ca_certs = ca_certs
-            self.ciphers = ciphers
-        # Can't use sock.type as other flags (such as SOCK_NONBLOCK) get
-        # mixed in.
-        if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM:
-            raise NotImplementedError("only stream sockets are supported")
-        if server_side and server_hostname:
-            raise ValueError("server_hostname can only be specified "
-                             "in client mode")
-        if self._context.check_hostname and not server_hostname:
-            raise ValueError("check_hostname requires server_hostname")
-        self.server_side = server_side
-        self.server_hostname = server_hostname
-        self.do_handshake_on_connect = do_handshake_on_connect
-        self.suppress_ragged_eofs = suppress_ragged_eofs
-        if sock is not None:
-            socket.__init__(self,
-                            family=sock.family,
-                            type=sock.type,
-                            proto=sock.proto,
-                            fileno=sock.fileno())
-            self.settimeout(sock.gettimeout())
-            sock.detach()
-        elif fileno is not None:
-            socket.__init__(self, fileno=fileno)
-        else:
-            socket.__init__(self, family=family, type=type, proto=proto)
-
-        # See if we are connected
-        try:
-            self.getpeername()
-        except OSError as e:
-            if e.errno != errno.ENOTCONN:
-                raise
-            connected = False
-        else:
-            connected = True
-
-        self._closed = False
-        self._sslobj = None
-        self._connected = connected
-        if connected:
-            # create the SSL object
-            try:
-                sslobj = self._context._wrap_socket(self, server_side,
-                                                    server_hostname)
-                self._sslobj = SSLObject(sslobj, owner=self)
-                if do_handshake_on_connect:
-                    timeout = self.gettimeout()
-                    if timeout == 0.0:
-                        # non-blocking
-                        raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-                    self.do_handshake()
-
-            except (OSError, ValueError):
-                self.close()
-                raise
-
-    @property
-    def context(self):
-        return self._context
-
-    @context.setter
-    def context(self, ctx):
-        self._context = ctx
-        self._sslobj.context = ctx
-
-    def dup(self):
-        raise NotImplemented("Can't dup() %s instances" %
-                             self.__class__.__name__)
-
-    def _checkClosed(self, msg=None):
-        # raise an exception here if you wish to check for spurious closes
-        pass
-
-    def _check_connected(self):
-        if not self._connected:
-            # getpeername() will raise ENOTCONN if the socket is really
-            # not connected; note that we can be connected even without
-            # _connected being set, e.g. if connect() first returned
-            # EAGAIN.
-            self.getpeername()
-
-    def read(self, len=1024, buffer=None):
-        """Read up to LEN bytes and return them.
-        Return zero-length string on EOF."""
-
-        self._checkClosed()
-        if not self._sslobj:
-            raise ValueError("Read on closed or unwrapped SSL socket.")
-        try:
-            return self._sslobj.read(len, buffer)
-        except SSLError as x:
-            if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs:
-                if buffer is not None:
-                    return 0
-                else:
-                    return b''
-            else:
-                raise
-
-    def write(self, data):
-        """Write DATA to the underlying SSL channel.  Returns
-        number of bytes of DATA actually transmitted."""
-
-        self._checkClosed()
-        if not self._sslobj:
-            raise ValueError("Write on closed or unwrapped SSL socket.")
-        return self._sslobj.write(data)
-
-    def getpeercert(self, binary_form=False):
-        """Returns a formatted version of the data in the
-        certificate provided by the other end of the SSL channel.
-        Return None if no certificate was provided, {} if a
-        certificate was provided, but not validated."""
-
-        self._checkClosed()
-        self._check_connected()
-        return self._sslobj.getpeercert(binary_form)
-
-    def selected_npn_protocol(self):
-        self._checkClosed()
-        if not self._sslobj or not _ssl.HAS_NPN:
-            return None
-        else:
-            return self._sslobj.selected_npn_protocol()
-
-    def selected_alpn_protocol(self):
-        self._checkClosed()
-        if not self._sslobj or not _ssl.HAS_ALPN:
-            return None
-        else:
-            return self._sslobj.selected_alpn_protocol()
-
-    def cipher(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        else:
-            return self._sslobj.cipher()
-
-    def shared_ciphers(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        return self._sslobj.shared_ciphers()
-
-    def compression(self):
-        self._checkClosed()
-        if not self._sslobj:
-            return None
-        else:
-            return self._sslobj.compression()
-
-    def send(self, data, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to send() on %s" %
-                    self.__class__)
-            return self._sslobj.write(data)
-        else:
-            return socket.send(self, data, flags)
-
-    def sendto(self, data, flags_or_addr, addr=None):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("sendto not allowed on instances of %s" %
-                             self.__class__)
-        elif addr is None:
-            return socket.sendto(self, data, flags_or_addr)
-        else:
-            return socket.sendto(self, data, flags_or_addr, addr)
-
-    def sendmsg(self, *args, **kwargs):
-        # Ensure programs don't send data unencrypted if they try to
-        # use this method.
-        raise NotImplementedError("sendmsg not allowed on instances of %s" %
-                                  self.__class__)
-
-    def sendall(self, data, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to sendall() on %s" %
-                    self.__class__)
-            amount = len(data)
-            count = 0
-            while (count < amount):
-                v = self.send(data[count:])
-                count += v
-            return amount
-        else:
-            return socket.sendall(self, data, flags)
-
-    def sendfile(self, file, offset=0, count=None):
-        """Send a file, possibly by using os.sendfile() if this is a
-        clear-text socket.  Return the total number of bytes sent.
-        """
-        if self._sslobj is None:
-            # os.sendfile() works with plain sockets only
-            return super().sendfile(file, offset, count)
-        else:
-            return self._sendfile_use_send(file, offset, count)
-
-    def recv(self, buflen=1024, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                    "non-zero flags not allowed in calls to recv() on %s" %
-                    self.__class__)
-            return self.read(buflen)
-        else:
-            return socket.recv(self, buflen, flags)
-
-    def recv_into(self, buffer, nbytes=None, flags=0):
-        self._checkClosed()
-        if buffer and (nbytes is None):
-            nbytes = len(buffer)
-        elif nbytes is None:
-            nbytes = 1024
-        if self._sslobj:
-            if flags != 0:
-                raise ValueError(
-                  "non-zero flags not allowed in calls to recv_into() on %s" %
-                  self.__class__)
-            return self.read(nbytes, buffer)
-        else:
-            return socket.recv_into(self, buffer, nbytes, flags)
-
-    def recvfrom(self, buflen=1024, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("recvfrom not allowed on instances of %s" %
-                             self.__class__)
-        else:
-            return socket.recvfrom(self, buflen, flags)
-
-    def recvfrom_into(self, buffer, nbytes=None, flags=0):
-        self._checkClosed()
-        if self._sslobj:
-            raise ValueError("recvfrom_into not allowed on instances of %s" %
-                             self.__class__)
-        else:
-            return socket.recvfrom_into(self, buffer, nbytes, flags)
-
-    def recvmsg(self, *args, **kwargs):
-        raise NotImplementedError("recvmsg not allowed on instances of %s" %
-                                  self.__class__)
-
-    def recvmsg_into(self, *args, **kwargs):
-        raise NotImplementedError("recvmsg_into not allowed on instances of "
-                                  "%s" % self.__class__)
-
-    def pending(self):
-        self._checkClosed()
-        if self._sslobj:
-            return self._sslobj.pending()
-        else:
-            return 0
-
-    def shutdown(self, how):
-        self._checkClosed()
-        self._sslobj = None
-        socket.shutdown(self, how)
-
-    def unwrap(self):
-        if self._sslobj:
-            s = self._sslobj.unwrap()
-            self._sslobj = None
-            return s
-        else:
-            raise ValueError("No SSL wrapper around " + str(self))
-
-    def _real_close(self):
-        self._sslobj = None
-        socket._real_close(self)
-
-    def do_handshake(self, block=False):
-        """Perform a TLS/SSL handshake."""
-        self._check_connected()
-        timeout = self.gettimeout()
-        try:
-            if timeout == 0.0 and block:
-                self.settimeout(None)
-            self._sslobj.do_handshake()
-        finally:
-            self.settimeout(timeout)
-
-    def _real_connect(self, addr, connect_ex):
-        if self.server_side:
-            raise ValueError("can't connect in server-side mode")
-        # Here we assume that the socket is client-side, and not
-        # connected at the time of the call.  We connect it, then wrap it.
-        if self._connected:
-            raise ValueError("attempt to connect already-connected SSLSocket!")
-        sslobj = self.context._wrap_socket(self, False, self.server_hostname)
-        self._sslobj = SSLObject(sslobj, owner=self)
-        try:
-            if connect_ex:
-                rc = socket.connect_ex(self, addr)
-            else:
-                rc = None
-                socket.connect(self, addr)
-            if not rc:
-                self._connected = True
-                if self.do_handshake_on_connect:
-                    self.do_handshake()
-            return rc
-        except (OSError, ValueError):
-            self._sslobj = None
-            raise
-
-    def connect(self, addr):
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-        self._real_connect(addr, False)
-
-    def connect_ex(self, addr):
-        """Connects to remote ADDR, and then wraps the connection in
-        an SSL channel."""
-        return self._real_connect(addr, True)
-
-    def accept(self):
-        """Accepts a new connection from a remote client, and returns
-        a tuple containing that new connection wrapped with a server-side
-        SSL channel, and the address of the remote client."""
-
-        newsock, addr = socket.accept(self)
-        newsock = self.context.wrap_socket(newsock,
-                    do_handshake_on_connect=self.do_handshake_on_connect,
-                    suppress_ragged_eofs=self.suppress_ragged_eofs,
-                    server_side=True)
-        return newsock, addr
-
-    def get_channel_binding(self, cb_type="tls-unique"):
-        """Get channel binding data for current connection.  Raise ValueError
-        if the requested `cb_type` is not supported.  Return bytes of the data
-        or None if the data is not available (e.g. before the handshake).
-        """
-        if self._sslobj is None:
-            return None
-        return self._sslobj.get_channel_binding(cb_type)
-
-    def version(self):
-        """
-        Return a string identifying the protocol version used by the
-        current SSL channel, or None if there is no established channel.
-        """
-        if self._sslobj is None:
-            return None
-        return self._sslobj.version()
-
-
-def wrap_socket(sock, keyfile=None, certfile=None,
-                server_side=False, cert_reqs=CERT_NONE,
-                ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-                do_handshake_on_connect=True,
-                suppress_ragged_eofs=True,
-                ciphers=None):
-
-    return SSLSocket(sock=sock, keyfile=keyfile, certfile=certfile,
-                     server_side=server_side, cert_reqs=cert_reqs,
-                     ssl_version=ssl_version, ca_certs=ca_certs,
-                     do_handshake_on_connect=do_handshake_on_connect,
-                     suppress_ragged_eofs=suppress_ragged_eofs,
-                     ciphers=ciphers)
-
-# some utility functions
-
-def cert_time_to_seconds(cert_time):
-    """Return the time in seconds since the Epoch, given the timestring
-    representing the "notBefore" or "notAfter" date from a certificate
-    in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale).
-
-    "notBefore" or "notAfter" dates must use UTC (RFC 5280).
-
-    Month is one of: Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec
-    UTC should be specified as GMT (see ASN1_TIME_print())
-    """
-    from time import strptime
-    from calendar import timegm
-
-    months = (
-        "Jan","Feb","Mar","Apr","May","Jun",
-        "Jul","Aug","Sep","Oct","Nov","Dec"
-    )
-    time_format = ' %d %H:%M:%S %Y GMT' # NOTE: no month, fixed GMT
-    try:
-        month_number = months.index(cert_time[:3].title()) + 1
-    except ValueError:
-        raise ValueError('time data %r does not match '
-                         'format "%%b%s"' % (cert_time, time_format))
-    else:
-        # found valid month
-        tt = strptime(cert_time[3:], time_format)
-        # return an integer, the previous mktime()-based implementation
-        # returned a float (fractional seconds are always zero here).
-        return timegm((tt[0], month_number) + tt[2:6])
-
-PEM_HEADER = "-----BEGIN CERTIFICATE-----"
-PEM_FOOTER = "-----END CERTIFICATE-----"
-
-def DER_cert_to_PEM_cert(der_cert_bytes):
-    """Takes a certificate in binary DER format and returns the
-    PEM version of it as a string."""
-
-    f = str(base64.standard_b64encode(der_cert_bytes), 'ASCII', 'strict')
-    return (PEM_HEADER + '\n' +
-            textwrap.fill(f, 64) + '\n' +
-            PEM_FOOTER + '\n')
-
-def PEM_cert_to_DER_cert(pem_cert_string):
-    """Takes a certificate in ASCII PEM format and returns the
-    DER-encoded version of it as a byte sequence"""
-
-    if not pem_cert_string.startswith(PEM_HEADER):
-        raise ValueError("Invalid PEM encoding; must start with %s"
-                         % PEM_HEADER)
-    if not pem_cert_string.strip().endswith(PEM_FOOTER):
-        raise ValueError("Invalid PEM encoding; must end with %s"
-                         % PEM_FOOTER)
-    d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)]
-    return base64.decodebytes(d.encode('ASCII', 'strict'))
-
-def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None):
-    """Retrieve the certificate from the server at the specified address,
-    and return it as a PEM-encoded string.
-    If 'ca_certs' is specified, validate the server cert against it.
-    If 'ssl_version' is specified, use it in the connection attempt."""
-
-    host, port = addr
-    if ca_certs is not None:
-        cert_reqs = CERT_REQUIRED
-    else:
-        cert_reqs = CERT_NONE
-    context = _create_stdlib_context(ssl_version,
-                                     cert_reqs=cert_reqs,
-                                     cafile=ca_certs)
-    with  create_connection(addr) as sock:
-        with context.wrap_socket(sock) as sslsock:
-            dercert = sslsock.getpeercert(True)
-    return DER_cert_to_PEM_cert(dercert)
-
-def get_protocol_name(protocol_code):
-    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -9,8 +9,6 @@
 
 def extension_suffixes(space):
     suffixes_w = []
-    # REVIEW this condition prevents pypy/tools/build_cffi_imports.py to succeed if
-    # pypy was compiled without cpyext module. why nore always append it?
     if space.config.objspace.usemodules.cpyext:
         suffixes_w.append(space.wrap(importing.get_so_extension(space)))
     return space.newlist(suffixes_w)

From pypy.commits at gmail.com  Mon Nov 28 11:02:59 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 08:02:59 -0800 (PST)
Subject: [pypy-commit] pypy default: kill this 'if 0' path for VMS
Message-ID: <583c5533.cf3fc20a.5b1c7.30f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88706:402299f61542
Date: 2016-11-28 17:02 +0100
http://bitbucket.org/pypy/pypy/changeset/402299f61542/

Log:	kill this 'if 0' path for VMS

diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -75,16 +75,6 @@
 
             return rffi.charpsize2str(rffi.cast(rffi.CCHARP, buf), n)
 
-elif 0:  # __VMS
-    from rpython.rlib.ropenssl import libssl_RAND_pseudo_bytes
-    def init_urandom():
-        pass
-
-    def urandom(context, n):
-        with rffi.scoped_alloc_buffer(n) as buf:
-            if libssl_RAND_pseudo_bytes(self.raw, n) < 0:
-                raise ValueError("RAND_pseudo_bytes")
-            return buf.str(n)
 else:  # Posix implementation
     @not_rpython
     def init_urandom():

From pypy.commits at gmail.com  Mon Nov 28 11:32:23 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 08:32:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: copy over _hashlib from pypy/modules
 to lib_pypy and start to rewrite first functions
Message-ID: <583c5c17.0b561c0a.d1ca8.0415@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88707:9c52653f210c
Date: 2016-11-28 17:31 +0100
http://bitbucket.org/pypy/pypy/changeset/9c52653f210c/

Log:	copy over _hashlib from pypy/modules to lib_pypy and start to
	rewrite first functions

diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
deleted file mode 100644
--- a/pypy/module/_hashlib/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._hashlib.interp_hashlib import (
-    algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC)
-
-
-class Module(MixedModule):
-    interpleveldefs = {
-        'new' : 'interp_hashlib.new',
-        }
-
-    appleveldefs = {
-        }
-
-    for name in algorithms:
-        interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name
-
-    if HAS_FAST_PKCS5_PBKDF2_HMAC:
-        interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac'
-
-    def startup(self, space):
-        w_meth_names = fetch_names(space)
-        space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names)
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
deleted file mode 100644
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ /dev/null
@@ -1,209 +0,0 @@
-from __future__ import with_statement
-
-from rpython.rlib import rgc, ropenssl
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rstring import StringBuilder
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.tool.sourcetools import func_renamer
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.thread.os_lock import Lock
-
-
-algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
-
-def hash_name_mapper_callback(obj_name, userdata):
-    if not obj_name:
-        return
-    # Ignore aliased names, they pollute the list and OpenSSL appears
-    # to have a its own definition of alias as the resulting list
-    # still contains duplicate and alternate names for several
-    # algorithms.
-    if rffi.cast(lltype.Signed, obj_name[0].c_alias):
-        return
-    try:
-        space = global_name_fetcher.space
-        w_name = space.wrap(rffi.charp2str(obj_name[0].c_name))
-        global_name_fetcher.meth_names.append(w_name)
-    except OperationError as e:
-        global_name_fetcher.w_error = e
-
-class NameFetcher:
-    def setup(self, space):
-        self.space = space
-        self.meth_names = []
-        self.w_error = None
-    def _cleanup_(self):
-        self.__dict__.clear()
-global_name_fetcher = NameFetcher()
-
-def fetch_names(space):
-    global_name_fetcher.setup(space)
-    ropenssl.init_digests()
-    ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH,
-                             hash_name_mapper_callback, None)
-    if global_name_fetcher.w_error:
-        raise global_name_fetcher.w_error
-    meth_names = global_name_fetcher.meth_names
-    global_name_fetcher.meth_names = None
-    return space.call_function(space.w_frozenset, space.newlist(meth_names))
-
-class W_Hash(W_Root):
-    NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-    ctx = NULL_CTX
-
-    def __init__(self, space, name, copy_from=NULL_CTX):
-        self.name = name
-        digest_type = self.digest_type_by_name(space)
-        self.digest_size = ropenssl.EVP_MD_size(digest_type)
-
-        # Allocate a lock for each HASH object.
-        # An optimization would be to not release the GIL on small requests,
-        # and use a custom lock only when needed.
-        self.lock = Lock(space)
-
-        ctx = ropenssl.EVP_MD_CTX_new()
-        if ctx is None:
-            raise MemoryError
-        rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
-        try:
-            if copy_from:
-                if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
-                    raise ValueError
-            else:
-                ropenssl.EVP_DigestInit(ctx, digest_type)
-            self.ctx = ctx
-        except:
-            ropenssl.EVP_MD_CTX_free(ctx)
-            raise
-        self.register_finalizer(space)
-
-    def _finalize_(self):
-        ctx = self.ctx
-        if ctx:
-            self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-            ropenssl.EVP_MD_CTX_free(ctx)
-
-    def digest_type_by_name(self, space):
-        digest_type = ropenssl.EVP_get_digestbyname(self.name)
-        if not digest_type:
-            raise oefmt(space.w_ValueError, "unknown hash function")
-        return digest_type
-
-    def descr_repr(self, space):
-        addrstring = self.getaddrstring(space)
-        return space.wrap("<%s HASH object at 0x%s>" % (
-            self.name, addrstring))
-
-    @unwrap_spec(string='bufferstr')
-    def update(self, space, string):
-        with rffi.scoped_nonmovingbuffer(string) as buf:
-            with self.lock:
-                # XXX try to not release the GIL for small requests
-                ropenssl.EVP_DigestUpdate(self.ctx, buf, len(string))
-
-    def copy(self, space):
-        "Return a copy of the hash object."
-        with self.lock:
-            return W_Hash(space, self.name, copy_from=self.ctx)
-
-    def digest(self, space):
-        "Return the digest value as a string of binary data."
-        digest = self._digest(space)
-        return space.newbytes(digest)
-
-    def hexdigest(self, space):
-        "Return the digest value as a string of hexadecimal digits."
-        digest = self._digest(space)
-        hexdigits = '0123456789abcdef'
-        result = StringBuilder(self.digest_size * 2)
-        for c in digest:
-            result.append(hexdigits[(ord(c) >> 4) & 0xf])
-            result.append(hexdigits[ ord(c)       & 0xf])
-        return space.wrap(result.build())
-
-    def get_digest_size(self, space):
-        return space.wrap(self.digest_size)
-
-    def get_block_size(self, space):
-        digest_type = self.digest_type_by_name(space)
-        block_size = ropenssl.EVP_MD_block_size(digest_type)
-        return space.wrap(block_size)
-
-    def get_name(self, space):
-        return space.wrap(self.name)
-
-    def _digest(self, space):
-        ctx = ropenssl.EVP_MD_CTX_new()
-        if ctx is None:
-            raise MemoryError
-        try:
-            with self.lock:
-                if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx):
-                    raise ValueError
-            digest_size = self.digest_size
-            with rffi.scoped_alloc_buffer(digest_size) as buf:
-                ropenssl.EVP_DigestFinal(ctx, buf.raw, None)
-                return buf.str(digest_size)
-        finally:
-            ropenssl.EVP_MD_CTX_free(ctx)
-
-
-W_Hash.typedef = TypeDef(
-    'HASH',
-    __repr__=interp2app(W_Hash.descr_repr),
-    update=interp2app(W_Hash.update),
-    copy=interp2app(W_Hash.copy),
-    digest=interp2app(W_Hash.digest),
-    hexdigest=interp2app(W_Hash.hexdigest),
-    #
-    digest_size=GetSetProperty(W_Hash.get_digest_size),
-    block_size=GetSetProperty(W_Hash.get_block_size),
-    name=GetSetProperty(W_Hash.get_name),
-)
-W_Hash.typedef.acceptable_as_base_class = False
-
- at unwrap_spec(name=str, string='bufferstr')
-def new(space, name, string=''):
-    w_hash = W_Hash(space, name)
-    w_hash.update(space, string)
-    return space.wrap(w_hash)
-
-# shortcut functions
-def make_new_hash(name, funcname):
-    @func_renamer(funcname)
-    @unwrap_spec(string='bufferstr')
-    def new_hash(space, string=''):
-        return new(space, name, string)
-    return new_hash
-
-for _name in algorithms:
-    _newname = 'new_%s' % (_name,)
-    globals()[_newname] = make_new_hash(_name, _newname)
-
-
-HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None
-if HAS_FAST_PKCS5_PBKDF2_HMAC:
-    @unwrap_spec(name=str, password='bytes', salt='bytes', rounds=int,
-                 w_dklen=WrappedDefault(None))
-    def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen):
-        digest = ropenssl.EVP_get_digestbyname(name)
-        if not digest:
-            raise oefmt(space.w_ValueError, "unknown hash function")
-        if space.is_w(w_dklen, space.w_None):
-            dklen = ropenssl.EVP_MD_size(digest)
-        else:
-            dklen = space.int_w(w_dklen)
-        if dklen < 1:
-            raise oefmt(space.w_ValueError,
-                        "key length must be greater than 0.")
-        with rffi.scoped_alloc_buffer(dklen) as buf:
-            r = ropenssl.PKCS5_PBKDF2_HMAC(
-                password, len(password), salt, len(salt), rounds, digest,
-                dklen, buf.raw)
-            if not r:
-                raise ValueError
-            return space.newbytes(buf.str(dklen))
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
deleted file mode 100644
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ /dev/null
@@ -1,100 +0,0 @@
-class AppTestHashlib:
-    spaceconfig = {
-        "usemodules": ['_hashlib', 'array', 'struct', 'binascii'],
-    }
-
-    def test_method_names(self):
-        import _hashlib
-        assert isinstance(_hashlib.openssl_md_meth_names, frozenset)
-        assert "md5" in _hashlib.openssl_md_meth_names
-
-    def test_simple(self):
-        import _hashlib
-        assert _hashlib.new('md5').__class__.__name__ == 'HASH'
-        assert len(_hashlib.new('md5').hexdigest()) == 32
-
-    def test_attributes(self):
-        import hashlib
-        for name, (expected_size, expected_block_size) in {
-                'md5': (16, 64),
-                'sha1': (20, 64),
-                'sha224': (28, 64),
-                'sha256': (32, 64),
-                'sha384': (48, 128),
-                'sha512': (64, 128),
-                }.items():
-            h = hashlib.new(name)
-            assert h.name == name
-            assert h.digest_size == expected_size
-            assert h.block_size == expected_block_size
-            #
-            h.update(b'abc')
-            h2 = h.copy()
-            h.update(b'def')
-            digest = h.digest()
-            hexdigest = h.hexdigest()
-            h2.update(b'd')
-            h2.update(b'ef')
-            assert digest    == h2.digest()
-            assert hexdigest == h2.hexdigest()
-            assert len(digest)    == h.digest_size
-            assert len(hexdigest) == h.digest_size * 2
-            c_digest    = digest
-            c_hexdigest = hexdigest
-
-
-    def test_shortcut(self):
-        import hashlib
-        assert repr(hashlib.md5()).startswith("<md5 HASH object")
-
-    def test_uppercase(self):
-        import _hashlib
-        h = _hashlib.new('MD5')
-        assert h.digest_size == 16
-        assert len(h.hexdigest()) == 32
-
-    def test_buffer(self):
-        import _hashlib, array
-        b = array.array('b', b'x' * 10)
-        h = _hashlib.new('md5', b)
-        h.update(b)
-        assert h.digest() == _hashlib.openssl_md5(b'x' * 20).digest()
-        _hashlib.openssl_sha1(b).digest()
-
-    def test_extra_algorithms(self):
-        expected_results = {
-            "md5": "bb649c83dd1ea5c9d9dec9a18df0ffe9",
-            "md4": "c275b8454684ea416b93d7a418b43176",
-            "mdc2": None,   # XXX find the correct expected value
-            "sha": "e2b0a8609b47c58e5d984c9ccfe69f9b654b032b",
-            "ripemd160": "cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc",
-            "whirlpool": ("1a22b79fe5afda02c63a25927193ed01dc718b74"
-                          "026e597608ce431f9c3d2c9e74a7350b7fbb7c5d"
-                          "4effe5d7a31879b8b7a10fd2f544c4ca268ecc6793923583"),
-            }
-        import _hashlib
-        test_string = b"Nobody inspects the spammish repetition"
-        for hash_name, expected in sorted(expected_results.items()):
-            try:
-                m = _hashlib.new(hash_name)
-            except ValueError as e:
-                print('skipped %s: %s' % (hash_name, e))
-                continue
-            m.update(test_string)
-            got = m.hexdigest()
-            assert got and type(got) is str and len(got) % 2 == 0
-            if expected is not None:
-                assert got == expected
-
-    def test_pbkdf2(self):
-        try:
-            from _hashlib import pbkdf2_hmac
-        except ImportError:
-            skip("Requires OpenSSL >= 1.1")
-        out = pbkdf2_hmac('sha1', b'password', b'salt', 1)
-        assert type(out) is bytes
-        assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex')
-        out = pbkdf2_hmac('sha1', b'password', b'salt', 2, None)
-        assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex')
-        raises(TypeError, pbkdf2_hmac, 'sha1', 'password', b'salt', 1)
-        raises(TypeError, pbkdf2_hmac, 'sha1', b'password', 'salt', 1)
diff --git a/pypy/module/_hashlib/test/test_ztranslation.py b/pypy/module/_hashlib/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/_hashlib/test/test_ztranslation.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-
-def test_checkmodule():
-    checkmodule('_hashlib')
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -18,6 +18,7 @@
     "lzma": "_lzma_build.py",
     "_decimal": "_decimal_build.py",
     "ssl": "_ssl_build.py",
+    # hashlib does not need to be built! It uses API calls from ssl
     "xx": None,    # for testing: 'None' should be completely ignored
     }
 

From pypy.commits at gmail.com  Mon Nov 28 11:35:02 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 28 Nov 2016 08:35:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: strip _hashlib from pypyoptions
Message-ID: <583c5cb6.e2acc20a.38e57.61b2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88708:accd3b5317bb
Date: 2016-11-28 17:34 +0100
http://bitbucket.org/pypy/pypy/changeset/accd3b5317bb/

Log:	strip _hashlib from pypyoptions

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -35,7 +35,7 @@
 working_modules.update([
     "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
-    "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
+    "zlib", "bz2", "struct", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
@@ -118,7 +118,6 @@
     "zlib"      : ["rpython.rlib.rzlib"],
     "bz2"       : ["pypy.module.bz2.interp_bz2"],
     "pyexpat"   : ["pypy.module.pyexpat.interp_pyexpat"],
-    "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
     "_vmprof"      : ["pypy.module._vmprof.interp_vmprof"],

From pypy.commits at gmail.com  Mon Nov 28 12:24:29 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:24:29 -0800 (PST)
Subject: [pypy-commit] pypy default: Provide
 rarithmetic.ovfcheck_int32_add/sub/mul, with a reasonable
Message-ID: <583c684d.8a29c20a.df6a.5c3f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88709:2b67d420c658
Date: 2016-11-28 18:23 +0100
http://bitbucket.org/pypy/pypy/changeset/2b67d420c658/

Log:	Provide rarithmetic.ovfcheck_int32_add/sub/mul, with a reasonable
	implementation (not special-cased by the JIT, but should generate
	only two extra instructions)

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -25,6 +25,10 @@
          lltype.Unsigned, widen it to lltype.Signed.
          Useful because the translator doesn't support
          arithmetic on the smaller types.
+ovfcheck_int32_add/sub/mul(x, y)
+         perform an add/sub/mul between two regular integers,
+         but assumes that they fit inside signed 32-bit ints
+         and raises OverflowError if the result no longer does
 
 These are meant to be erased by translation, r_uint
 in the process should mark unsigned values, ovfcheck should
@@ -796,6 +800,47 @@
         return longlong2float(rffi.cast(rffi.LONGLONG, res))
     return rffi.cast(T, res)
 
+if sys.maxint == 2147483647:
+    def ovfcheck_int32_add(x, y):
+        return ovfcheck(x + y)
+    def ovfcheck_int32_sub(x, y):
+        return ovfcheck(x - y)
+    def ovfcheck_int32_mul(x, y):
+        return ovfcheck(x * y)
+else:
+    def ovfcheck_int32_add(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x + y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_sub(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x - y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_mul(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x * y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
 
 # String parsing support
 # ---------------------------
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -675,3 +675,11 @@
     assert a ^ r_uint32(42) == "a^42"
     assert r_uint32(42) ** a == "42**a"
     assert a ** r_uint32(42) == "a**42"
+
+def test_ovfcheck_int32():
+    assert ovfcheck_int32_add(-2**30, -2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_add, 2**30, 2**30)
+    assert ovfcheck_int32_sub(-2**30, 2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
+    assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)

From pypy.commits at gmail.com  Mon Nov 28 12:33:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:33:11 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix for test_sysmodule
Message-ID: <583c6a57.6a5cc20a.a0487.5aec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88710:43ec8832c98f
Date: 2016-11-28 18:32 +0100
http://bitbucket.org/pypy/pypy/changeset/43ec8832c98f/

Log:	Fix for test_sysmodule

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -3,7 +3,7 @@
 import py
 
 from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.sourcetools import rpython_wrapper
@@ -1221,7 +1221,8 @@
         x = jit.conditional_call_elidable(self.cache, _compute_and_cache, ...)
 
     """
-    if we_are_jitted():
+    if we_are_translated() and we_are_jitted():
+        #^^^ the occasional test patches we_are_jitted() to True
         return _jit_conditional_call_value(value, function, *args)
     else:
         if isinstance(value, int):

From pypy.commits at gmail.com  Mon Nov 28 12:34:18 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:34:18 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <583c6a9a.90a81c0a.b1ffa.37b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88711:c8b4b78af01a
Date: 2016-11-28 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/c8b4b78af01a/

Log:	hg merge default

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -74,6 +74,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
@@ -109,6 +122,12 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
     if args:
         vals = []
         for name in args:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,7 @@
 
 Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
 call this function" returning a result.
+
+.. branch: desc-specialize
+
+Refactor FunctionDesc.specialize() and related code (RPython annotator).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -302,6 +302,12 @@
         elif config.objspace.usemodules.pypyjit:
             config.translation.jit = True
 
+        if config.objspace.usemodules.cpyext:
+            if config.translation.gc != 'incminimark':
+                raise Exception("The 'cpyext' module requires the 'incminimark'"
+                                " GC.  You need either 'targetpypystandalone.py"
+                                " --withoutmod-cpyext' or '--gc=incminimark'")
+
         config.translating = True
 
         import translate
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -559,6 +559,7 @@
                                           space.wrap(msg))
         return OperationError(exc, w_error)
 
+ at specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
                   w_exception_class=None):
     assert isinstance(e, OSError)
@@ -586,8 +587,8 @@
         w_error = space.call_function(exc, space.wrap(errno),
                                       space.wrap(msg))
     return OperationError(exc, w_error)
-wrap_oserror2._annspecialcase_ = 'specialize:arg(3)'
 
+ at specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
                  w_exception_class=None):
     if filename is not None:
@@ -598,7 +599,6 @@
         return wrap_oserror2(space, e, None,
                              exception_name=exception_name,
                              w_exception_class=w_exception_class)
-wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
@@ -7,7 +8,6 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.sliceobject import unwrap_start_stop
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -124,13 +124,13 @@
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         if can_be_None and w_obj is None:
             return None
         if not isinstance(w_obj, RequiredClass):
             raise TypeError
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def getarg_w(self, code, w_obj):    # for retrieving buffers
         return FakeBuffer(w_obj)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -35,8 +35,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -1622,9 +1620,8 @@
     miniglobals = {'__name__':    __name__, # for module name propagation
                    }
     exec source.compile() in miniglobals
-    call_external_function = miniglobals['cpy_call_external']
+    call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
     call_external_function._dont_inline_ = True
-    call_external_function._annspecialcase_ = 'specialize:ll'
     call_external_function._gctransformer_hint_close_stack_ = True
     # don't inline, as a hack to guarantee that no GC pointer is alive
     # anywhere in call_external_function
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,7 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import fatalerror
 
 
 #________________________________________________________
@@ -192,6 +193,8 @@
     rawrefcount.create_link_pypy(w_obj, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 def from_ref(space, ref):
     """
     Finds the interpreter object corresponding to the given reference.  If the
@@ -202,7 +205,23 @@
         return None
     w_obj = rawrefcount.to_obj(W_Root, ref)
     if w_obj is not None:
-        return w_obj
+        if w_obj is not w_marker_deallocating:
+            return w_obj
+        fatalerror(
+            "*** Invalid usage of a dying CPython object ***\n"
+            "\n"
+            "cpyext, the emulation layer, detected that while it is calling\n"
+            "an object's tp_dealloc, the C code calls back a function that\n"
+            "tries to recreate the PyPy version of the object.  Usually it\n"
+            "means that tp_dealloc calls some general PyXxx() API.  It is\n"
+            "a dangerous and potentially buggy thing to do: even in CPython\n"
+            "the PyXxx() function could, in theory, cause a reference to the\n"
+            "object to be taken and stored somewhere, for an amount of time\n"
+            "exceeding tp_dealloc itself.  Afterwards, the object will be\n"
+            "freed, making that reference point to garbage.\n"
+            ">>> PyPy could contain some workaround to still work if\n"
+            "you are lucky, but it is not done so far; better fix the bug in\n"
+            "the CPython extension.")
 
     # This reference is not yet a real interpreter object.
     # Realize it.
@@ -233,7 +252,8 @@
 INTERPLEVEL_API['as_pyobj'] = as_pyobj
 
 def pyobj_has_w_obj(pyobj):
-    return rawrefcount.to_obj(W_Root, pyobj) is not None
+    w_obj = rawrefcount.to_obj(W_Root, pyobj)
+    return w_obj is not None and w_obj is not w_marker_deallocating
 INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj)
 
 
@@ -335,6 +355,7 @@
     pto = obj.c_ob_type
     #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \
     #      "'s type which is", rffi.charp2str(pto.c_tp_name)
+    rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -4,11 +4,11 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.unicodehelper import encode_utf8
 from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -208,6 +208,7 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):
@@ -221,7 +222,6 @@
                 return w_some_obj()
             self._wrap_not_rpython(x)
         return w_some_obj()
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
@@ -314,10 +314,10 @@
         is_root(w_complex)
         return 1.1, 2.2
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     def decode_index(self, w_index_or_slice, seqlength):
         is_root(w_index_or_slice)
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -9,9 +9,9 @@
 
 def make_checker():
     check = []
+    @specialize.memo()
     def see():
         check.append(True)
-    see._annspecialcase_ = 'specialize:memo'
     return see, check
 
 def test_wrap_interp2app():
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -2,12 +2,12 @@
 import sys
 
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import INT_MAX
 from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -206,6 +206,10 @@
         pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
+
+    # XXX for Darwin running old versions of CPython 2.7.x
+    sysconfig.get_config_vars()
+
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler)  # XXX
     objects = []
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -2,6 +2,7 @@
 
 import types
 from collections import defaultdict
+from contextlib import contextmanager
 
 from rpython.tool.ansi_print import AnsiLogger
 from rpython.tool.pairtype import pair
@@ -83,22 +84,17 @@
         annmodel.TLS.check_str_without_nul = (
             self.translator.config.translation.check_str_without_nul)
 
-        flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
+        with self.using_policy(policy):
+            flowgraph, inputs_s = self.get_call_parameters(function, args_s)
 
         if main_entry_point:
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
-    def get_call_parameters(self, function, args_s, policy):
-        desc = self.bookkeeper.getdesc(function)
-        prevpolicy = self.policy
-        self.policy = policy
-        self.bookkeeper.enter(None)
-        try:
+    def get_call_parameters(self, function, args_s):
+        with self.bookkeeper.at_position(None):
+            desc = self.bookkeeper.getdesc(function)
             return desc.get_call_parameters(args_s)
-        finally:
-            self.bookkeeper.leave()
-            self.policy = prevpolicy
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
@@ -107,21 +103,29 @@
             # XXX hack
             annmodel.TLS.check_str_without_nul = (
                 self.translator.config.translation.check_str_without_nul)
-        graph, inputcells = self.get_call_parameters(function, args_s, policy)
-        self.build_graph_types(graph, inputcells, complete_now=False)
-        self.complete_helpers(policy)
+        with self.using_policy(policy):
+            graph, inputcells = self.get_call_parameters(function, args_s)
+            self.build_graph_types(graph, inputcells, complete_now=False)
+            self.complete_helpers()
         return graph
 
-    def complete_helpers(self, policy):
-        saved = self.policy, self.added_blocks
-        self.policy = policy
+    def complete_helpers(self):
+        saved = self.added_blocks
+        self.added_blocks = {}
         try:
-            self.added_blocks = {}
             self.complete()
             # invoke annotation simplifications for the new blocks
             self.simplify(block_subset=self.added_blocks)
         finally:
-            self.policy, self.added_blocks = saved
+            self.added_blocks = saved
+
+    @contextmanager
+    def using_policy(self, policy):
+        """A context manager that temporarily replaces the annotator policy"""
+        old_policy = self.policy
+        self.policy = policy
+        yield
+        self.policy = old_policy
 
     def build_graph_types(self, flowgraph, inputcells, complete_now=True):
         checkgraph(flowgraph)
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -9,6 +9,7 @@
 from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
+from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.model import (
     SomeOrderedDict, SomeString, SomeChar, SomeFloat, unionof, SomeInstance,
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
@@ -21,6 +22,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
+from rpython.annotator.specialize import memo
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
@@ -358,7 +360,7 @@
             return self.descs[obj_key]
         except KeyError:
             if isinstance(pyobj, types.FunctionType):
-                result = description.FunctionDesc(self, pyobj)
+                result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
                     raise Exception("ClassDesc for object not supported")
@@ -403,6 +405,23 @@
             self.descs[obj_key] = result
             return result
 
+    def newfuncdesc(self, pyfunc):
+        name = pyfunc.__name__
+        if hasattr(pyfunc, '_generator_next_method_of_'):
+            from rpython.flowspace.argument import Signature
+            signature = Signature(['entry'])     # haaaaaack
+            defaults = ()
+        else:
+            signature = cpython_code_signature(pyfunc.func_code)
+            defaults = pyfunc.func_defaults
+        # get the specializer based on the tag of the 'pyobj'
+        # (if any), according to the current policy
+        tag = getattr(pyfunc, '_annspecialcase_', None)
+        specializer = self.annotator.policy.get_specializer(tag)
+        if specializer is memo:
+            return description.MemoDesc(self, pyfunc, name, signature, defaults, specializer)
+        return description.FunctionDesc(self, pyfunc, name, signature, defaults, specializer)
+
     def getfrozen(self, pyobj):
         return description.FrozenDesc(self, pyobj)
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -608,7 +608,7 @@
             if mixin:
                 # make a new copy of the FunctionDesc for this class,
                 # but don't specialize further for all subclasses
-                funcdesc = FunctionDesc(self.bookkeeper, value)
+                funcdesc = self.bookkeeper.newfuncdesc(value)
                 self.classdict[name] = funcdesc
                 return
             # NB. if value is, say, AssertionError.__init__, then we
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -3,11 +3,10 @@
 from rpython.annotator.signature import (
     enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import FunctionGraph
-from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, s_ImpossibleValue
+from rpython.annotator.model import AnnotatorError, s_ImpossibleValue, unionof
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -117,7 +116,6 @@
         self.s_value = s_ImpossibleValue    # union of possible values
 
     def update(self, other):
-        from rpython.annotator.model import unionof
         self.descs.update(other.descs)
         self.read_locations.update(other.read_locations)
         self.s_value = unionof(self.s_value, other.s_value)
@@ -192,24 +190,12 @@
 class FunctionDesc(Desc):
     knowntype = types.FunctionType
 
-    def __init__(self, bookkeeper, pyobj=None,
-                 name=None, signature=None, defaults=None,
+    def __init__(self, bookkeeper, pyobj, name, signature, defaults,
                  specializer=None):
         super(FunctionDesc, self).__init__(bookkeeper, pyobj)
-        if name is None:
-            name = pyobj.func_name
-        if signature is None:
-            if hasattr(pyobj, '_generator_next_method_of_'):
-                from rpython.flowspace.argument import Signature
-                signature = Signature(['entry'])     # haaaaaack
-                defaults = ()
-            else:
-                signature = cpython_code_signature(pyobj.func_code)
-        if defaults is None:
-            defaults = pyobj.func_defaults
         self.name = name
         self.signature = signature
-        self.defaults = defaults or ()
+        self.defaults = defaults if defaults is not None else ()
         # 'specializer' is a function with the following signature:
         #      specializer(funcdesc, args_s) => graph
         #                                 or => s_result (overridden/memo cases)
@@ -288,12 +274,43 @@
                 getattr(self.bookkeeper, "position_key", None) is not None):
             _, block, i = self.bookkeeper.position_key
             op = block.operations[i]
-        if self.specializer is None:
-            # get the specializer based on the tag of the 'pyobj'
-            # (if any), according to the current policy
-            tag = getattr(self.pyobj, '_annspecialcase_', None)
-            policy = self.bookkeeper.annotator.policy
-            self.specializer = policy.get_specializer(tag)
+        self.normalize_args(inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
+
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
+        # Some specializations may break the invariant of returning
+        # annotations that are always more general than the previous time.
+        # We restore it here:
+        result = unionof(result, s_previous_result)
+        return result
+
+    def normalize_args(self, inputs_s):
+        """
+        Canonicalize argument annotations into the exact parameter
+        annotations of a specific specialized graph.
+
+        Note: this method has no return value but mutates its argument instead.
+        """
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
@@ -304,39 +321,9 @@
                 from rpython.annotator.signature import Sig
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
-            enforceargs(self, inputcells)  # can modify inputcells in-place
+            enforceargs(self, inputs_s)  # can modify inputs_s in-place
         if signature:
-            enforce_signature_args(self, signature[0], inputcells)  # mutates inputcells
-        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
-            return self.specializer(self, inputcells, op)
-        else:
-            return self.specializer(self, inputcells)
-
-    def pycall(self, whence, args, s_previous_result, op=None):
-        inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
-        # Some specializations may break the invariant of returning
-        # annotations that are always more general than the previous time.
-        # We restore it here:
-        from rpython.annotator.model import unionof
-        result = unionof(result, s_previous_result)
-        return result
+            enforce_signature_args(self, signature[0], inputs_s)  # mutates inputs_s
 
     def get_graph(self, args, op):
         inputs_s = self.parse_arguments(args)
@@ -405,6 +392,16 @@
 
             return s_sigs
 
+class MemoDesc(FunctionDesc):
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        s_result = self.specialize(inputcells, op)
+        if isinstance(s_result, FunctionGraph):
+            s_result = s_result.getreturnvar().annotation
+        s_result = unionof(s_result, s_previous_result)
+        return s_result
+
+
 class MethodDesc(Desc):
     knowntype = types.MethodType
 
diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py
--- a/rpython/annotator/specialize.py
+++ b/rpython/annotator/specialize.py
@@ -3,11 +3,13 @@
 
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import Block, Link, Variable, SpaceOperation
+from rpython.flowspace.model import Block, Link, Variable
 from rpython.flowspace.model import checkgraph
 from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.flowspace.argument import Signature
+from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
+from rpython.annotator.model import unionof
 
 def flatten_star_args(funcdesc, args_s):
     argnames, vararg, kwarg = funcdesc.signature
@@ -127,7 +129,6 @@
     def finish(self):
         if self.do_not_process:
             return
-        from rpython.annotator.model import unionof
         assert self.graph is None, "MemoTable already finished"
         # list of which argument positions can take more than one value
         example_args, example_value = self.table.iteritems().next()
@@ -246,34 +247,36 @@
             args_s.append(unionof(*values_s))
         annotator.addpendinggraph(self.graph, args_s)
 
+def all_values(s):
+    """Return the exhaustive list of possible values matching annotation `s`.
 
-def memo(funcdesc, arglist_s):
-    from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
-    from rpython.annotator.model import unionof
+    Raises `AnnotatorError` if no such (reasonably small) finite list exists.
+    """
+    if s.is_constant():
+        return [s.const]
+    elif isinstance(s, SomePBC):
+        values = []
+        assert not s.can_be_None, "memo call: cannot mix None and PBCs"
+        for desc in s.descriptions:
+            if desc.pyobj is None:
+                raise annmodel.AnnotatorError(
+                    "memo call with a class or PBC that has no "
+                    "corresponding Python object (%r)" % (desc,))
+            values.append(desc.pyobj)
+        return values
+    elif isinstance(s, SomeImpossibleValue):
+        return []
+    elif isinstance(s, SomeBool):
+        return [False, True]
+    else:
+        raise annmodel.AnnotatorError("memo call: argument must be a class "
+                                        "or a frozen PBC, got %r" % (s,))
+
+def memo(funcdesc, args_s):
     # call the function now, and collect possible results
-    argvalues = []
-    for s in arglist_s:
-        if s.is_constant():
-            values = [s.const]
-        elif isinstance(s, SomePBC):
-            values = []
-            assert not s.can_be_None, "memo call: cannot mix None and PBCs"
-            for desc in s.descriptions:
-                if desc.pyobj is None:
-                    raise annmodel.AnnotatorError(
-                        "memo call with a class or PBC that has no "
-                        "corresponding Python object (%r)" % (desc,))
-                values.append(desc.pyobj)
-        elif isinstance(s, SomeImpossibleValue):
-            return s    # we will probably get more possible args later
-        elif isinstance(s, SomeBool):
-            values = [False, True]
-        else:
-            raise annmodel.AnnotatorError("memo call: argument must be a class "
-                                          "or a frozen PBC, got %r" % (s,))
-        argvalues.append(values)
+
     # the list of all possible tuples of arguments to give to the memo function
-    possiblevalues = cartesian_product(argvalues)
+    possiblevalues = cartesian_product([all_values(s_arg) for s_arg in args_s])
 
     # a MemoTable factory -- one MemoTable per family of arguments that can
     # be called together, merged via a UnionFind.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2936,6 +2936,12 @@
         self._pyobj(pyobject).ob_pypy_link = objint
         # there is no rrc_o_dict
 
+    def rawrefcount_mark_deallocating(self, gcobj, pyobject):
+        ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        obj = llmemory.cast_ptr_to_adr(gcobj)   # should be a prebuilt obj
+        objint = llmemory.cast_adr_to_int(obj, "symbolic")
+        self._pyobj(pyobject).ob_pypy_link = objint
+
     def rawrefcount_from_obj(self, gcobj):
         obj = llmemory.cast_ptr_to_adr(gcobj)
         if self.is_in_nursery(obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -476,6 +476,10 @@
                 GCClass.rawrefcount_create_link_pyobj,
                 [s_gc, s_gcref, SomeAddress()],
                 annmodel.s_None)
+            self.rawrefcount_mark_deallocating = getfn(
+                GCClass.rawrefcount_mark_deallocating,
+                [s_gc, s_gcref, SomeAddress()],
+                annmodel.s_None)
             self.rawrefcount_from_obj_ptr = getfn(
                 GCClass.rawrefcount_from_obj, [s_gc, s_gcref], SomeAddress(),
                 inline = True)
@@ -1281,6 +1285,14 @@
                   [self.rawrefcount_create_link_pyobj_ptr, self.c_const_gc,
                    v_gcobj, v_pyobject])
 
+    def gct_gc_rawrefcount_mark_deallocating(self, hop):
+        [v_gcobj, v_pyobject] = hop.spaceop.args
+        assert v_gcobj.concretetype == llmemory.GCREF
+        assert v_pyobject.concretetype == llmemory.Address
+        hop.genop("direct_call",
+                  [self.rawrefcount_mark_deallocating, self.c_const_gc,
+                   v_gcobj, v_pyobject])
+
     def gct_gc_rawrefcount_from_obj(self, hop):
         [v_gcobj] = hop.spaceop.args
         assert v_gcobj.concretetype == llmemory.GCREF
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -3,7 +3,7 @@
 import py
 
 from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.sourcetools import rpython_wrapper
@@ -1221,7 +1221,8 @@
         x = jit.conditional_call_elidable(self.cache, _compute_and_cache, ...)
 
     """
-    if we_are_jitted():
+    if we_are_translated() and we_are_jitted():
+        #^^^ the occasional test patches we_are_jitted() to True
         return _jit_conditional_call_value(value, function, *args)
     else:
         if isinstance(value, int):
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -25,6 +25,10 @@
          lltype.Unsigned, widen it to lltype.Signed.
          Useful because the translator doesn't support
          arithmetic on the smaller types.
+ovfcheck_int32_add/sub/mul(x, y)
+         perform an add/sub/mul between two regular integers,
+         but assumes that they fit inside signed 32-bit ints
+         and raises OverflowError if the result no longer does
 
 These are meant to be erased by translation, r_uint
 in the process should mark unsigned values, ovfcheck should
@@ -796,6 +800,47 @@
         return longlong2float(rffi.cast(rffi.LONGLONG, res))
     return rffi.cast(T, res)
 
+if sys.maxint == 2147483647:
+    def ovfcheck_int32_add(x, y):
+        return ovfcheck(x + y)
+    def ovfcheck_int32_sub(x, y):
+        return ovfcheck(x - y)
+    def ovfcheck_int32_mul(x, y):
+        return ovfcheck(x * y)
+else:
+    def ovfcheck_int32_add(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x + y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_sub(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x - y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_mul(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x * y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
 
 # String parsing support
 # ---------------------------
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -36,6 +36,7 @@
     _pypy2ob = {}
     _pypy2ob_rev = {}
     _d_list = []
+    _d_marker = None
     _dealloc_trigger_callback = dealloc_trigger_callback
 
 @not_rpython
@@ -62,6 +63,14 @@
     _o_list.append(ob)
 
 @not_rpython
+def mark_deallocating(marker, ob):
+    """mark the PyObject as deallocating, by storing 'marker'
+    inside its ob_pypy_link field"""
+    assert ob._obj not in _pypy2ob_rev
+    assert not ob.c_ob_pypy_link
+    ob.c_ob_pypy_link = _build_pypy_link(marker)
+
+ at not_rpython
 def from_obj(OB_PTR_TYPE, p):
     ob = _pypy2ob.get(p)
     if ob is None:
@@ -221,7 +230,7 @@
 
 
 class Entry(ExtRegistryEntry):
-    _about_ = (create_link_pypy, create_link_pyobj)
+    _about_ = (create_link_pypy, create_link_pyobj, mark_deallocating)
 
     def compute_result_annotation(self, s_p, s_ob):
         pass
@@ -231,6 +240,8 @@
             name = 'gc_rawrefcount_create_link_pypy'
         elif self.instance is create_link_pyobj:
             name = 'gc_rawrefcount_create_link_pyobj'
+        elif self.instance is mark_deallocating:
+            name = 'gc_rawrefcount_mark_deallocating'
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -13,6 +13,7 @@
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.history import ConstInt, ConstFloat
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import r_longlong
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import compute_unique_id, always_inline
 from rpython.rlib.objectmodel import we_are_translated, specialize
@@ -113,6 +114,7 @@
 
 @always_inline
 def encode_le_64bit(val):
+    val = r_longlong(val)     # force 64-bit, even on 32-bit
     return ''.join([chr((val >> 0) & 0xff),
                     chr((val >> 8) & 0xff),
                     chr((val >> 16) & 0xff),
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -75,16 +75,6 @@
 
             return rffi.charpsize2str(rffi.cast(rffi.CCHARP, buf), n)
 
-elif 0:  # __VMS
-    from rpython.rlib.ropenssl import libssl_RAND_pseudo_bytes
-    def init_urandom():
-        pass
-
-    def urandom(context, n):
-        with rffi.scoped_alloc_buffer(n) as buf:
-            if libssl_RAND_pseudo_bytes(self.raw, n) < 0:
-                raise ValueError("RAND_pseudo_bytes")
-            return buf.str(n)
 else:  # Posix implementation
     @not_rpython
     def init_urandom():
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -126,13 +126,14 @@
 // how we'd get the PC (using StackWalk64?)
 //    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
 
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
+// #include "base/logging.h"   // for RAW_LOG
+// #ifndef HAVE_CYGWIN_SIGNAL_H
+// typedef int ucontext_t;
+// #endif
 
 intptr_t GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  fprintf(stderr, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -675,3 +675,11 @@
     assert a ^ r_uint32(42) == "a^42"
     assert r_uint32(42) ** a == "42**a"
     assert a ** r_uint32(42) == "a**42"
+
+def test_ovfcheck_int32():
+    assert ovfcheck_int32_add(-2**30, -2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_add, 2**30, 2**30)
+    assert ovfcheck_int32_sub(-2**30, 2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
+    assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -212,6 +212,15 @@
         assert rawrefcount.to_obj(W_Root, ob) == p
         lltype.free(ob, flavor='raw')
 
+    def test_mark_deallocating(self):
+        ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+        w_marker = W_Root(42)
+        rawrefcount.mark_deallocating(w_marker, ob)
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        rawrefcount._collect()
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        lltype.free(ob, flavor='raw')
+
 
 class TestTranslated(StandaloneTests):
 
@@ -222,6 +231,7 @@
         state.seen = []
         def dealloc_trigger():
             state.seen.append(1)
+        w_marker = W_Root(-1)
 
         def make_p():
             p = W_Root(42)
@@ -257,6 +267,13 @@
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
+            if rawrefcount.to_obj(W_Root, ob) is not None:
+                print "to_obj(dead) is not None?"
+                return 1
+            rawrefcount.mark_deallocating(w_marker, ob)
+            if rawrefcount.to_obj(W_Root, ob) is not w_marker:
+                print "to_obj(marked-dead) is not w_marker"
+                return 1
             print "OK!"
             lltype.free(ob, flavor='raw')
             return 0
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -138,11 +138,12 @@
         # get the graph of the mix-level helper ll_function and prepare it for
         # being annotated.  Annotation and RTyping should be done in a single shot
         # at the end with finish().
-        graph, args_s = self.rtyper.annotator.get_call_parameters(
-            ll_function, args_s, policy = self.policy)
+        ann = self.rtyper.annotator
+        with ann.using_policy(self.policy):
+            graph, args_s = ann.get_call_parameters(ll_function, args_s)
         for v_arg, s_arg in zip(graph.getargs(), args_s):
-            self.rtyper.annotator.setbinding(v_arg, s_arg)
-        self.rtyper.annotator.setbinding(graph.getreturnvar(), s_result)
+            ann.setbinding(v_arg, s_arg)
+        ann.setbinding(graph.getreturnvar(), s_result)
         #self.rtyper.annotator.annotated[graph.returnblock] = graph
         self.pending.append((ll_function, graph, args_s, s_result))
         return graph
@@ -224,16 +225,17 @@
         bk = ann.bookkeeper
         translator = ann.translator
         original_graph_count = len(translator.graphs)
-        for ll_function, graph, args_s, s_result in self.pending:
-            # mark the return block as already annotated, because the return var
-            # annotation was forced in getgraph() above.  This prevents temporary
-            # less general values reaching the return block from crashing the
-            # annotator (on the assert-that-new-binding-is-not-less-general).
-            ann.annotated[graph.returnblock] = graph
-            s_function = bk.immutablevalue(ll_function)
-            bk.emulate_pbc_call(graph, s_function, args_s)
-            self.newgraphs.add(graph)
-        ann.complete_helpers(self.policy)
+        with ann.using_policy(self.policy):
+            for ll_function, graph, args_s, s_result in self.pending:
+                # mark the return block as already annotated, because the return var
+                # annotation was forced in getgraph() above.  This prevents temporary
+                # less general values reaching the return block from crashing the
+                # annotator (on the assert-that-new-binding-is-not-less-general).
+                ann.annotated[graph.returnblock] = graph
+                s_function = bk.immutablevalue(ll_function)
+                bk.emulate_pbc_call(graph, s_function, args_s)
+                self.newgraphs.add(graph)
+            ann.complete_helpers()
         for ll_function, graph, args_s, s_result in self.pending:
             s_real_result = ann.binding(graph.getreturnvar())
             if s_real_result != s_result:
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -959,6 +959,9 @@
     def op_gc_rawrefcount_create_link_pypy(self, *args):
         raise NotImplementedError("gc_rawrefcount_create_link_pypy")
 
+    def op_gc_rawrefcount_mark_deallocating(self, *args):
+        raise NotImplementedError("gc_rawrefcount_mark_deallocating")
+
     def op_do_malloc_fixedsize(self):
         raise NotImplementedError("do_malloc_fixedsize")
     def op_do_malloc_fixedsize_clear(self):
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -384,7 +384,6 @@
 def _sizeof_none(TYPE):
     assert not TYPE._is_varsize()
     return ItemOffset(TYPE)
-_sizeof_none._annspecialcase_ = 'specialize:memo'
 
 @specialize.memo()
 def _internal_array_field(TYPE):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -490,6 +490,7 @@
     'gc_rawrefcount_init':              LLOp(),
     'gc_rawrefcount_create_link_pypy':  LLOp(),
     'gc_rawrefcount_create_link_pyobj': LLOp(),
+    'gc_rawrefcount_mark_deallocating': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
 
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -7,7 +7,7 @@
 from rpython.rtyper.lltypesystem.llmemory import itemoffsetof
 from rpython.rtyper.llannotation import lltype_to_annotation
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, specialize
 from rpython.rlib.objectmodel import keepalive_until_here, enforceargs
 from rpython.rlib import rarithmetic, rgc
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -74,7 +74,7 @@
 def llexternal(name, args, result, _callable=None,
                compilation_info=ExternalCompilationInfo(),
                sandboxsafe=False, releasegil='auto',
-               _nowrapper=False, calling_conv='c',
+               _nowrapper=False, calling_conv=None,
                elidable_function=False, macro=None,
                random_effects_on_gcobjs='auto',
                save_err=RFFI_ERR_NONE):
@@ -96,7 +96,17 @@
                 we consider that the function is really short-running and
                 don't bother releasing the GIL.  An explicit True or False
                 overrides this logic.
+
+    calling_conv: if 'unknown' or 'win', the C function is not directly seen
+                  by the JIT.  If 'c', it can be seen (depending on
+                  releasegil=False).  For tests only, or if _nowrapper,
+                  it defaults to 'c'.
     """
+    if calling_conv is None:
+        if sys.platform == 'win32' and not _nowrapper:
+            calling_conv = 'unknown'
+        else:
+            calling_conv = 'c'
     if _callable is not None:
         assert callable(_callable)
     ext_type = lltype.FuncType(args, result)
@@ -107,7 +117,8 @@
             _callable = generate_macro_wrapper(
                 name, macro, ext_type, compilation_info)
         else:
-            _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv)
+            _callable = ll2ctypes.LL2CtypesCallable(ext_type,
+                'c' if calling_conv == 'unknown' else calling_conv)
     else:
         assert macro is None, "'macro' is useless if you specify '_callable'"
     if elidable_function:
@@ -205,11 +216,17 @@
     else:
         # if we don't have to invoke the GIL handling, we can just call
         # the low-level function pointer carelessly
-        if macro is None and save_err == RFFI_ERR_NONE:
+        # ...well, unless it's a macro, in which case we still have
+        # to hide it from the JIT...
+        need_wrapper = (macro is not None or save_err != RFFI_ERR_NONE)
+        # ...and unless we're on Windows and the calling convention is
+        # 'win' or 'unknown'
+        if calling_conv != 'c':
+            need_wrapper = True
+        #
+        if not need_wrapper:
             call_external_function = funcptr
         else:
-            # ...well, unless it's a macro, in which case we still have
-            # to hide it from the JIT...
             argnames = ', '.join(['a%d' % i for i in range(len(args))])
             source = py.code.Source("""
                 def call_external_function(%(argnames)s):
@@ -308,10 +325,6 @@
     # for debugging, stick ll func ptr to that
     wrapper._ptr = funcptr
     wrapper = func_with_new_name(wrapper, name)
-
-    if calling_conv != "c":
-        wrapper = jit.dont_look_inside(wrapper)
-
     return wrapper
 
 
@@ -1288,10 +1301,12 @@
 c_memcpy = llexternal("memcpy",
             [VOIDP, VOIDP, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            calling_conv='c',
         )
 c_memset = llexternal("memset",
             [VOIDP, lltype.Signed, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            calling_conv='c',
         )
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -362,9 +362,9 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         funcdesc, = s_pbc.descriptions
-        args = simple_args(args_s)
         with bk.at_position(None):
-            graph = funcdesc.get_graph(args, op)
+            argspec = simple_args(args_s)
+            graph = funcdesc.get_graph(argspec, op)
         llfn = self.rtyper.getcallable(graph)
         return inputconst(typeOf(llfn), llfn)
 
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -846,28 +846,29 @@
         rtyper = self.rtyper
         args_s = []
         newargs_v = []
-        for v in args_v:
-            if v.concretetype is Void:
-                s_value = rtyper.annotation(v)
-                if s_value is None:
-                    s_value = annmodel.s_None
-                if not s_value.is_constant():
-                    raise TyperError("non-constant variable of type Void")
-                if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
-                    raise TyperError("non-PBC Void argument: %r", (s_value,))
-                args_s.append(s_value)
-            else:
-                args_s.append(lltype_to_annotation(v.concretetype))
-            newargs_v.append(v)
+        with rtyper.annotator.using_policy(rtyper.lowlevel_ann_policy):
+            for v in args_v:
+                if v.concretetype is Void:
+                    s_value = rtyper.annotation(v)
+                    if s_value is None:
+                        s_value = annmodel.s_None
+                    if not s_value.is_constant():
+                        raise TyperError("non-constant variable of type Void")
+                    if not isinstance(s_value, (annmodel.SomePBC, annmodel.SomeNone)):
+                        raise TyperError("non-PBC Void argument: %r", (s_value,))
+                    args_s.append(s_value)
+                else:
+                    args_s.append(lltype_to_annotation(v.concretetype))
+                newargs_v.append(v)
 
-        self.rtyper.call_all_setups()  # compute ForwardReferences now
+            self.rtyper.call_all_setups()  # compute ForwardReferences now
 
-        # hack for bound methods
-        if hasattr(ll_function, 'im_func'):
-            bk = rtyper.annotator.bookkeeper
-            args_s.insert(0, bk.immutablevalue(ll_function.im_self))
-            newargs_v.insert(0, inputconst(Void, ll_function.im_self))
-            ll_function = ll_function.im_func
+            # hack for bound methods
+            if hasattr(ll_function, 'im_func'):
+                bk = rtyper.annotator.bookkeeper
+                args_s.insert(0, bk.immutablevalue(ll_function.im_self))
+                newargs_v.insert(0, inputconst(Void, ll_function.im_self))
+                ll_function = ll_function.im_func
 
         graph = annotate_lowlevel_helper(rtyper.annotator, ll_function, args_s,
                                          rtyper.lowlevel_ann_policy)

From pypy.commits at gmail.com  Mon Nov 28 12:36:33 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:36:33 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583c6b21.43e61c0a.513b1.2de8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88712:f6c2cac88fa1
Date: 2016-11-28 18:35 +0100
http://bitbucket.org/pypy/pypy/changeset/f6c2cac88fa1/

Log:	fix test

diff --git a/lib-python/3/test/test_complex.py b/lib-python/3/test/test_complex.py
--- a/lib-python/3/test/test_complex.py
+++ b/lib-python/3/test/test_complex.py
@@ -310,7 +310,7 @@
         self.assertRaises(TypeError, float, 5+3j)
         self.assertRaises(ValueError, complex, "")
         self.assertRaises(TypeError, complex, None)
-        self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None)
+        self.assertRaisesRegex(TypeError, " 'NoneType'", complex, None)
         self.assertRaises(ValueError, complex, "\0")
         self.assertRaises(ValueError, complex, "3\09")
         self.assertRaises(TypeError, complex, "1", "2")

From pypy.commits at gmail.com  Mon Nov 28 12:45:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:45:24 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: b"%c" % bytearray(b"X")
Message-ID: <583c6d34.542e1c0a.d6cc.2e09@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88713:a9aa848c5cb2
Date: 2016-11-28 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/a9aa848c5cb2/

Log:	b"%c" % bytearray(b"X")

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -483,22 +483,24 @@
                                     "character code not in range(256)")
                     self.std_wp(s)
                 return
-            if space.isinstance_w(w_value, space.w_str):
-                s = space.str_w(w_value)
+            if not do_unicode:
+                if space.isinstance_w(w_value, space.w_str):
+                    s = space.str_w(w_value)
+                elif space.isinstance_w(w_value, space.w_bytearray):
+                    s = w_value.buffer_w(space, 0).as_str()
+                else:
+                    s = ''
                 if len(s) == 1:
                     self.std_wp(s)
                     return
-            elif space.isinstance_w(w_value, space.w_unicode):
-                if not do_unicode:
-                    raise NeedUnicodeFormattingError
-                ustr = space.unicode_w(w_value)
-                if len(ustr) == 1:
-                    self.std_wp(ustr)
-                    return
-            if do_unicode:
+                raise oefmt(space.w_TypeError, "%c requires int or single byte")
+            else:
+                if space.isinstance_w(w_value, space.w_unicode):
+                    ustr = space.unicode_w(w_value)
+                    if len(ustr) == 1:
+                        self.std_wp(ustr)
+                        return
                 raise oefmt(space.w_TypeError, "%c requires int or char")
-            else:
-                raise oefmt(space.w_TypeError, "%c requires int or single byte")
 
         def fmt_b(self, w_value):
             space = self.space
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -178,6 +178,7 @@
         raises(TypeError, '%c'.__mod__, ("bla",))
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))
+        raises(TypeError, '%c'.__mod__, b'A')
 
     def test___int__index__(self):
         class MyInt(object):
@@ -368,6 +369,7 @@
         assert b"<%c>" % 48 == b"<0>"
         assert b"<%c>" % b"?" == b"<?>"
         raises(TypeError, 'b"<%c>" % "?"')
+        assert b"<%c>" % bytearray(b"?") == b"<?>"
 
     def test_bytes_bytes(self):
         assert b"<%b>" % b"123" == b"<123>"
@@ -421,6 +423,8 @@
         assert bytearray(b"<%c>") % 48 == bytearray(b"<0>")
         assert bytearray(b"<%c>") % b"?" == bytearray(b"<?>")
         raises(TypeError, 'bytearray(b"<%c>") % "?"')
+        assert bytearray(b"<%c>") % bytearray(b"?") == bytearray(b"<?>")
+        raises(TypeError, 'bytearray(b"<%c>") % memoryview(b"X")')
 
     def test_bytes_bytes(self):
         assert bytearray(b"<%b>") % b"123" == bytearray(b"<123>")

From pypy.commits at gmail.com  Mon Nov 28 12:50:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:50:44 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: '%b' in unicode strings is not supported
Message-ID: <583c6e74.12ad1c0a.4a861.3953@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88714:481c5fb368c8
Date: 2016-11-28 18:50 +0100
http://bitbucket.org/pypy/pypy/changeset/481c5fb368c8/

Log:	'%b' in unicode strings is not supported

diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -503,6 +503,8 @@
                 raise oefmt(space.w_TypeError, "%c requires int or char")
 
         def fmt_b(self, w_value):
+            if do_unicode:
+                self.unknown_fmtchar()
             space = self.space
             # cpython explicitly checks for bytes & bytearray
             if space.isinstance_w(w_value, space.w_bytes):
@@ -524,7 +526,7 @@
                 return
 
             raise oefmt(space.w_TypeError,
-                    "requires bytes, or an object that" \
+                    "requires bytes, or an object that "
                     "implements __bytes__, not '%T'", w_value)
 
     return StringFormatter
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -321,6 +321,10 @@
         f = 4
         raises(ValueError, '"%\u1234" % (f,)')
 
+    def test_invalid_b_with_unicode(self):
+        raises(ValueError, '"%b" % b"A"')
+        raises(ValueError, '"%b" % 42')
+
     def test_formatting_huge_precision(self):
         prec = 2**31
         format_string = u"%.{}f".format(prec)
@@ -370,6 +374,10 @@
         assert b"<%c>" % b"?" == b"<?>"
         raises(TypeError, 'b"<%c>" % "?"')
         assert b"<%c>" % bytearray(b"?") == b"<?>"
+        class X:
+            def __bytes__(self):
+                return b'5'
+        raises(TypeError, 'b"<%c>" % X()')
 
     def test_bytes_bytes(self):
         assert b"<%b>" % b"123" == b"<123>"

From pypy.commits at gmail.com  Mon Nov 28 12:52:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 09:52:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583c6ed7.4673c20a.e67f3.677b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88715:b6249dffd811
Date: 2016-11-28 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/b6249dffd811/

Log:	fix test

diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -1013,6 +1013,7 @@
             self.assertNotEqual(wr(), None)
         else:
             self.fail("RecursionError not raised")
+        gc_collect()
         self.assertEqual(wr(), None)
 
     def test_errno_ENOTDIR(self):

From pypy.commits at gmail.com  Mon Nov 28 13:09:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 10:09:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add OSError.filename2
Message-ID: <583c72d3.c64bc20a.a1c99.6772@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88716:8335b93c9873
Date: 2016-11-28 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/8335b93c9873/

Log:	add OSError.filename2

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -446,6 +446,7 @@
         self.w_errno = space.w_None
         self.w_strerror = space.w_None
         self.w_filename = space.w_None
+        self.w_filename2 = space.w_None
         self.written = -1  # only for BlockingIOError.
         W_BaseException.__init__(self, space)
 
@@ -471,15 +472,17 @@
 
     @staticmethod
     def _parse_init_args(space, args_w):
-        if 2 <= len(args_w) <= 3:
+        if 2 <= len(args_w) <= 5:
             w_errno = args_w[0]
             w_strerror = args_w[1]
-            if len(args_w) == 3:
+            w_filename = None
+            w_filename2 = None
+            if len(args_w) > 2:
                 w_filename = args_w[2]
-            else:
-                w_filename = None
-            return w_errno, w_strerror, w_filename
-        return None, None, None
+                if len(args_w) > 4:
+                    w_filename2 = args_w[4]
+            return w_errno, w_strerror, w_filename, w_filename2
+        return None, None, None, None
 
     @staticmethod
     def descr_new(space, w_subtype, __args__):
@@ -487,11 +490,12 @@
         w_errno = None
         w_strerror = None
         w_filename = None
+        w_filename2 = None
         if not W_OSError._use_init(space, w_subtype):
             if kwds_w:
                 raise oefmt(space.w_TypeError,
                             "OSError does not take keyword arguments")
-            (w_errno, w_strerror, w_filename,
+            (w_errno, w_strerror, w_filename, w_filename2
              ) = W_OSError._parse_init_args(space, args_w)
         if (not space.is_none(w_errno) and
             space.is_w(w_subtype, space.gettypeobject(W_OSError.typedef))):
@@ -509,7 +513,8 @@
         exc = space.allocate_instance(W_OSError, w_subtype)
         W_OSError.__init__(exc, space)
         if not W_OSError._use_init(space, w_subtype):
-            exc._init_error(space, args_w, w_errno, w_strerror, w_filename)
+            exc._init_error(space, args_w, w_errno, w_strerror, w_filename,
+                            w_filename2)
         return space.wrap(exc)
 
     def descr_init(self, space, __args__):
@@ -520,11 +525,13 @@
         if kwds_w:
             raise oefmt(space.w_TypeError,
                         "OSError does not take keyword arguments")
-        (w_errno, w_strerror, w_filename
+        (w_errno, w_strerror, w_filename, w_filename2
          ) = W_OSError._parse_init_args(space, args_w)
-        self._init_error(space, args_w, w_errno, w_strerror, w_filename)
+        self._init_error(space, args_w, w_errno, w_strerror, w_filename,
+                         w_filename2)
 
-    def _init_error(self, space, args_w, w_errno, w_strerror, w_filename):
+    def _init_error(self, space, args_w, w_errno, w_strerror, w_filename,
+                    w_filename2):
         W_BaseException.descr_init(self, space, args_w)
         if w_errno:
             self.w_errno = w_errno
@@ -540,17 +547,21 @@
                     self.w_filename = w_filename
             else:
                 self.w_filename = w_filename
+                if not space.is_none(w_filename2):
+                    self.w_filename2 = w_filename2
                 # filename is removed from the args tuple (for compatibility
                 # purposes, see test_exceptions.py)
                 self.args_w = [w_errno, w_strerror]
 
     # since we rebind args_w, we need special reduce, grump
     def descr_reduce(self, space):
+        extra = []
         if not space.is_w(self.w_filename, space.w_None):
-            lst = [self.getclass(space), space.newtuple(
-                self.args_w + [self.w_filename])]
-        else:
-            lst = [self.getclass(space), space.newtuple(self.args_w)]
+            extra.append(self.w_filename)
+            if not space.is_w(self.w_filename2, space.w_None):
+                extra.append(space.w_None)
+                extra.append(self.w_filename2)
+        lst = [self.getclass(space), space.newtuple(self.args_w + extra)]
         if self.w_dict is not None and space.is_true(self.w_dict):
             lst = lst + [self.w_dict]
         return space.newtuple(lst)
@@ -561,6 +572,12 @@
             errno = space.unicode_w(space.str(self.w_errno))
             strerror = space.unicode_w(space.str(self.w_strerror))
             if not space.is_w(self.w_filename, space.w_None):
+                if not space.is_w(self.w_filename2, space.w_None):
+                    return space.wrap(u"[Errno %s] %s: %s -> %s" % (
+                        errno,
+                        strerror,
+                        space.unicode_w(space.repr(self.w_filename)),
+                        space.unicode_w(space.repr(self.w_filename2))))
                 return space.wrap(u"[Errno %s] %s: %s" % (
                     errno,
                     strerror,
@@ -590,6 +607,7 @@
     errno    = readwrite_attrproperty_w('w_errno',    W_OSError),
     strerror = readwrite_attrproperty_w('w_strerror', W_OSError),
     filename = readwrite_attrproperty_w('w_filename', W_OSError),
+    filename2= readwrite_attrproperty_w('w_filename2',W_OSError),
     characters_written = GetSetProperty(W_OSError.descr_get_written,
                                         W_OSError.descr_set_written),
     )
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -374,6 +374,23 @@
         e = SubOSErrorWithNew("some message", baz="baz")
         assert e.baz == "baz"
         assert e.args == ("some message",)
+        assert e.filename is None
+        assert e.filename2 is None
+
+    def test_oserror_3_args(self):
+        e = OSError(42, "bar", "baz")
+        assert e.args == (42, "bar")
+        assert e.filename == "baz"
+        assert e.filename2 is None
+        assert str(e) == "[Errno 42] bar: 'baz'"
+
+    def test_oserror_5_args(self):
+        # NB. argument 4 is only parsed on Windows
+        e = OSError(42, "bar", "baz", None, "bok")
+        assert e.args == (42, "bar")
+        assert e.filename == "baz"
+        assert e.filename2 == "bok"
+        assert str(e) == "[Errno 42] bar: 'baz' -> 'bok'"
 
     # Check the heuristic for print & exec covers significant cases
     # As well as placing some limits on false positives

From pypy.commits at gmail.com  Mon Nov 28 13:24:00 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 10:24:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Generate OSError with a filename2 from
 the posix module
Message-ID: <583c7640.624fc20a.7862c.764a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88717:b3e1b9a536b3
Date: 2016-11-28 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/b3e1b9a536b3/

Log:	Generate OSError with a filename2 from the posix module

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -561,7 +561,7 @@
 
 @specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
-                  w_exception_class=None):
+                  w_exception_class=None, w_filename2=None):
     assert isinstance(e, OSError)
 
     if _WINDOWS and isinstance(e, WindowsError):
@@ -581,8 +581,13 @@
     else:
         exc = w_exception_class
     if w_filename is not None:
-        w_error = space.call_function(exc, space.wrap(errno),
-                                      space.wrap(msg), w_filename)
+        if w_filename2 is not None:
+            w_error = space.call_function(exc, space.wrap(errno),
+                                          space.wrap(msg), w_filename,
+                                          space.w_None, w_filename2)
+        else:
+            w_error = space.call_function(exc, space.wrap(errno),
+                                          space.wrap(msg), w_filename)
     else:
         w_error = space.call_function(exc, space.wrap(errno),
                                       space.wrap(msg))
@@ -590,15 +595,17 @@
 
 @specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
-                 w_exception_class=None):
+                 w_exception_class=None, filename2=None):
+    w_filename = None
+    w_filename2 = None
     if filename is not None:
-        return wrap_oserror2(space, e, space.wrap(filename),
-                             exception_name=exception_name,
-                             w_exception_class=w_exception_class)
-    else:
-        return wrap_oserror2(space, e, None,
-                             exception_name=exception_name,
-                             w_exception_class=w_exception_class)
+        w_filename = space.wrap(filename)
+        if filename2 is not None:
+            w_filename2 = space.wrap(filename2)
+    return wrap_oserror2(space, e, w_filename,
+                         exception_name=exception_name,
+                         w_exception_class=w_exception_class,
+                         w_filename2=w_filename2)
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1008,7 +1008,7 @@
         else:
             dispatch_filename_2(rposix.rename)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
@@ -1115,7 +1115,7 @@
     rposix.kill(os.getpid(), signal.SIGABRT)
 
 @unwrap_spec(
-    src='fsencode', dst='fsencode',
+    src='fsencode', dst='fsencode',  # <- simpler: link() is never on Windows
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
     follow_symlinks=bool)
 def link(
@@ -1144,7 +1144,7 @@
         else:
             rposix.link(src, dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, filename=src, filename2=dst)
 
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
@@ -1171,7 +1171,7 @@
         else:
             dispatch_filename_2(rposix.symlink)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
 
 
 @unwrap_spec(
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1214,6 +1214,18 @@
         self.posix.RTLD_GLOBAL
         self.posix.RTLD_LOCAL
 
+    def test_error_message(self):
+        e = raises(OSError, self.posix.open, 'nonexistentfile1', 0)
+        assert str(e.value).endswith(": 'nonexistentfile1'")
+
+        e = raises(OSError, self.posix.link, 'nonexistentfile1', 'bok')
+        assert str(e.value).endswith(": 'nonexistentfile1' -> 'bok'")
+        e = raises(OSError, self.posix.rename, 'nonexistentfile1', 'bok')
+        assert str(e.value).endswith(": 'nonexistentfile1' -> 'bok'")
+
+        e = raises(OSError, self.posix.symlink, 'bok', '/nonexistentdir/boz')
+        assert str(e.value).endswith(": 'bok' -> '/nonexistentdir/boz'")
+
 
 class AppTestEnvironment(object):
     def setup_class(cls):

From pypy.commits at gmail.com  Mon Nov 28 13:53:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 10:53:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Rename the 'signal' module: it is now
 called '_signal' and we have
Message-ID: <583c7d15.820bc30a.f79f4.7f73@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88718:31a78c0851ed
Date: 2016-11-28 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/31a78c0851ed/

Log:	Rename the 'signal' module: it is now called '_signal' and we have
	lib-python/3/signal.py.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -595,7 +595,7 @@
     # set up the Ctrl-C => KeyboardInterrupt signal handler, if the
     # signal module is available
     try:
-        import signal
+        import _signal as signal
     except ImportError:
         pass
     else:
diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py
--- a/pypy/module/signal/__init__.py
+++ b/pypy/module/signal/__init__.py
@@ -3,6 +3,8 @@
 import os
 
 class Module(MixedModule):
+    applevel_name = '_signal'
+
     interpleveldefs = {
         'signal':              'interp_signal.signal',
         'getsignal':           'interp_signal.getsignal',
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -14,7 +14,7 @@
     def test_checksignals(self):
         space = self.space
         w_received = space.appexec([], """():
-            import signal
+            import _signal as signal
             received = []
             def myhandler(signum, frame):
                 received.append(signum)
@@ -40,7 +40,7 @@
     }
 
     def setup_class(cls):
-        cls.w_signal = cls.space.getbuiltinmodule('signal')
+        cls.w_signal = cls.space.getbuiltinmodule('_signal')
         cls.w_temppath = cls.space.wrap(
             str(py.test.ensuretemp("signal").join("foo.txt")))
         cls.w_appdirect = cls.space.wrap(cls.runappdirect)
@@ -92,7 +92,7 @@
         """
         Test that signal.signal returns SIG_DFL if that is the current handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
@@ -105,7 +105,7 @@
         """
         Test that signal.signal returns SIG_IGN if that is the current handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
@@ -119,7 +119,7 @@
         Test that signal.signal returns a Python object if one is the current
         handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
         def installed(*a):
             pass
 
@@ -134,7 +134,7 @@
         """
         Test that signal.getsignal returns the currently installed handler.
         """
-        from signal import getsignal, signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import getsignal, signal, SIGINT, SIG_DFL, SIG_IGN
 
         def handler(*a):
             pass
@@ -153,7 +153,7 @@
 
     def test_check_signum(self):
         import sys
-        from signal import getsignal, signal, NSIG
+        from _signal import getsignal, signal, NSIG
 
         # signum out of range fails
         raises(ValueError, getsignal, NSIG)
@@ -166,7 +166,7 @@
 
     def test_alarm(self):
         try:
-            from signal import alarm, signal, SIG_DFL, SIGALRM
+            from _signal import alarm, signal, SIG_DFL, SIGALRM
         except:
             skip('no alarm on this platform')
         import time
@@ -186,7 +186,7 @@
 
     def test_set_wakeup_fd(self):
         try:
-            import signal, posix, fcntl
+            import _signal as signal, posix, fcntl
         except ImportError:
             skip('cannot import posix or fcntl')
         def myhandler(signum, frame):
@@ -222,13 +222,13 @@
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
     def test_set_wakeup_fd_invalid(self):
-        import signal
+        import _signal as signal
         with open(self.temppath, 'wb') as f:
             fd = f.fileno()
         raises(ValueError, signal.set_wakeup_fd, fd)
 
     def test_siginterrupt(self):
-        import signal, os, time
+        import _signal as signal, os, time
         if not hasattr(signal, 'siginterrupt'):
             skip('non siginterrupt in signal')
         signum = signal.SIGUSR1
@@ -267,7 +267,7 @@
 
     def test_alarm_raise(self):
         try:
-            from signal import alarm, signal, SIG_DFL, SIGALRM
+            from _signal import alarm, signal, SIG_DFL, SIGALRM
         except ImportError:
             skip("no SIGALRM on this platform")
         import _socket
@@ -299,7 +299,7 @@
             py.test.skip("Unix only")
 
     def test_itimer_real(self):
-        import signal
+        import _signal as signal
 
         def sig_alrm(*args):
             self.called = True
@@ -316,7 +316,7 @@
         assert self.called
 
     def test_itimer_exc(self):
-        import signal
+        import _signal as signal
 
         raises(signal.ItimerError, signal.setitimer, -1, 0)
 
@@ -324,7 +324,7 @@
     spaceconfig = dict(usemodules=['signal', 'thread', 'time'])
 
     def test_pthread_kill(self):
-        import signal
+        import _signal as signal
         import _thread
         signum = signal.SIGUSR1
         def handler(signum, frame):
@@ -334,7 +334,7 @@
         raises(ZeroDivisionError, signal.pthread_kill, tid, signum)
         
     def test_sigwait(self):
-        import signal
+        import _signal as signal
         def handler(signum, frame):
             1/0
         signal.signal(signal.SIGALRM, handler)
@@ -343,7 +343,7 @@
         assert received == signal.SIGALRM
         
     def test_sigmask(self):
-        import signal, posix
+        import _signal as signal, posix
         signum1 = signal.SIGUSR1
         signum2 = signal.SIGUSR2
 

From pypy.commits at gmail.com  Mon Nov 28 14:56:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 11:56:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Try to fix Weak{Key,
 Value}Dictionary.__len__()
Message-ID: <583c8bdf.0a4cc20a.25f32.9573@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88719:338b6c45ed55
Date: 2016-11-28 20:22 +0100
http://bitbucket.org/pypy/pypy/changeset/338b6c45ed55/

Log:	Try to fix Weak{Key,Value}Dictionary.__len__()

diff --git a/lib-python/3/weakref.py b/lib-python/3/weakref.py
--- a/lib-python/3/weakref.py
+++ b/lib-python/3/weakref.py
@@ -147,7 +147,14 @@
         del self.data[key]
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data.values():
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, key):
         try:
@@ -376,11 +383,18 @@
         return self.data[ref(key)]
 
     def __len__(self):
-        if self._dirty_len and self._pending_removals:
-            # self._pending_removals may still contain keys which were
-            # explicitly removed, we have to scrub them (see issue #21173).
-            self._scrub_removals()
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        if self._dirty_len and self._pending_removals:
+#            # self._pending_removals may still contain keys which were
+#            # explicitly removed, we have to scrub them (see issue #21173).
+#            self._scrub_removals()
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __repr__(self):
         return "<%s at %#x>" % (self.__class__.__name__, id(self))

From pypy.commits at gmail.com  Mon Nov 28 14:56:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Mon, 28 Nov 2016 11:56:17 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: also fix WeakSet.__len__
Message-ID: <583c8be1.d32f1c0a.ea5b4.52e8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88720:8891136318b8
Date: 2016-11-28 20:27 +0100
http://bitbucket.org/pypy/pypy/changeset/8891136318b8/

Log:	also fix WeakSet.__len__

diff --git a/lib-python/3/_weakrefset.py b/lib-python/3/_weakrefset.py
--- a/lib-python/3/_weakrefset.py
+++ b/lib-python/3/_weakrefset.py
@@ -65,7 +65,14 @@
                     yield item
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, item):
         try:
diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py
--- a/lib-python/3/test/test_weakref.py
+++ b/lib-python/3/test/test_weakref.py
@@ -1103,7 +1103,7 @@
         # Keep an iterator alive
         it = dct.items()
         try:
-            next(it)
+            print(next(it))
         except StopIteration:
             pass
         del items
@@ -1112,7 +1112,10 @@
         del it
         gc.collect()
         gc.collect()
+        print(list(dct.items()))
         n2 = len(dct)
+        print(len(dct))
+        print(weakref)
         # one item may be kept alive inside the iterator
         self.assertIn(n1, (0, 1))
         self.assertEqual(n2, 0)

From pypy.commits at gmail.com  Tue Nov 29 03:12:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 00:12:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583d388a.2aa9c20a.88c0a.4fea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88721:f21619a0a9f0
Date: 2016-11-29 09:12 +0100
http://bitbucket.org/pypy/pypy/changeset/f21619a0a9f0/

Log:	fix test

diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -510,6 +510,7 @@
             fut.add_done_callback(lambda *args: self.loop.stop())
             self.loop.run_forever()
             fut = None # Trigger Future.__del__ or futures._TracebackLogger
+            support.gc_collect()
             if PY34:
                 # Future.__del__ in Python 3.4 logs error with
                 # an actual exception context

From pypy.commits at gmail.com  Tue Nov 29 03:14:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 00:14:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583d38d0.d5091c0a.b441e.6e06@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88722:2b71379daaac
Date: 2016-11-29 09:13 +0100
http://bitbucket.org/pypy/pypy/changeset/2b71379daaac/

Log:	fix test

diff --git a/lib-python/3/test/test_asyncio/test_futures.py b/lib-python/3/test/test_asyncio/test_futures.py
--- a/lib-python/3/test/test_asyncio/test_futures.py
+++ b/lib-python/3/test/test_asyncio/test_futures.py
@@ -238,6 +238,7 @@
         fut.set_exception(RuntimeError('boom'))
         del fut
         test_utils.run_briefly(self.loop)
+        support.gc_collect()
         self.assertTrue(m_log.error.called)
 
     @mock.patch('asyncio.base_events.logger')

From pypy.commits at gmail.com  Tue Nov 29 03:55:53 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 00:55:53 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: test and fix a warning
Message-ID: <583d4299.973f1c0a.5c182.7f05@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88723:2958f07abe78
Date: 2016-11-29 09:55 +0100
http://bitbucket.org/pypy/pypy/changeset/2958f07abe78/

Log:	test and fix a warning

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -20,7 +20,8 @@
         self.running = False
         self._name = name           # may be null, use get_name()
         self._qualname = qualname   # may be null, use get_qualname()
-        if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
+        if (isinstance(self, Coroutine)    # XXX would be cool not to need this
+                or self.pycode.co_flags & CO_YIELD_INSIDE_TRY):
             self.register_finalizer(self.space)
         self.saved_operr = None
 
@@ -385,7 +386,7 @@
            self.frame.last_instr == -1:
             space = self.space
             msg = u"coroutine '%s' was never awaited" % self.get_qualname()
-            space.warn(space.w_RuntimeWarning, space.wrap(msg))
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
         GeneratorOrCoroutine._finalize_(self)
 
 
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -160,3 +160,20 @@
         else:
             assert False, "should have raised"
         """
+
+    def test_runtime_warning(self): """
+        import gc, warnings
+        async def foobaz():
+            pass
+        with warnings.catch_warnings(record=True) as l:
+            foobaz()
+            gc.collect()
+            gc.collect()
+            gc.collect()
+
+        assert len(l) == 1, repr(l)
+        w = l[0].message
+        assert isinstance(w, RuntimeWarning)
+        assert str(w).startswith("coroutine ")
+        assert str(w).endswith("foobaz' was never awaited")
+        """

From pypy.commits at gmail.com  Tue Nov 29 04:02:04 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 01:02:04 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <583d440c.44e61c0a.5110b.8262@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88724:865fcea6b0e0
Date: 2016-11-29 10:01 +0100
http://bitbucket.org/pypy/pypy/changeset/865fcea6b0e0/

Log:	fix tests

diff --git a/lib-python/3/unittest/test/test_assertions.py b/lib-python/3/unittest/test/test_assertions.py
--- a/lib-python/3/unittest/test/test_assertions.py
+++ b/lib-python/3/unittest/test/test_assertions.py
@@ -2,6 +2,7 @@
 import warnings
 import weakref
 import unittest
+from test.support import gc_collect
 from itertools import product
 
 
@@ -124,8 +125,10 @@
                     self.foo()
 
         Foo("test_functional").run()
+        gc_collect()
         self.assertIsNone(wr())
         Foo("test_with").run()
+        gc_collect()
         self.assertIsNone(wr())
 
     def testAssertNotRegex(self):
diff --git a/lib-python/3/unittest/test/test_case.py b/lib-python/3/unittest/test/test_case.py
--- a/lib-python/3/unittest/test/test_case.py
+++ b/lib-python/3/unittest/test/test_case.py
@@ -18,7 +18,7 @@
     TestEquality, TestHashing, LoggingResult, LegacyLoggingResult,
     ResultWithNoStartTestRunStopTestRun
 )
-from test.support import captured_stderr
+from test.support import captured_stderr, gc_collect
 
 
 log_foo = logging.getLogger('foo')
@@ -1737,6 +1737,7 @@
         for method_name in ('test1', 'test2'):
             testcase = TestCase(method_name)
             testcase.run()
+            gc_collect()
             self.assertEqual(MyException.ninstance, 0)
 
 

From pypy.commits at gmail.com  Tue Nov 29 04:07:01 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 01:07:01 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583d4535.46bb1c0a.1c422.c58a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88725:e4e91dd9fe93
Date: 2016-11-29 10:06 +0100
http://bitbucket.org/pypy/pypy/changeset/e4e91dd9fe93/

Log:	fix test

diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py
--- a/lib-python/3/test/test_sys.py
+++ b/lib-python/3/test/test_sys.py
@@ -811,7 +811,12 @@
             ref = AtExit()
         """
         rc, stdout, stderr = assert_python_ok('-c', code)
-        self.assertEqual(stdout.rstrip(), b'True')
+        if test.support.check_impl_detail(cpython=True):
+            self.assertEqual(stdout.rstrip(), b'True')
+        else:
+            # the __del__ method may or may not have been called
+            # in other Python implementations
+            self.assertIn(stdout.rstrip(), {b'True', b''})
 
 
 @test.support.cpython_only

From pypy.commits at gmail.com  Tue Nov 29 04:12:50 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 01:12:50 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Catch and wrap the RPython-level OSError
Message-ID: <583d4692.8a29c20a.df6a.6f16@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88726:aa4ad227978a
Date: 2016-11-29 10:12 +0100
http://bitbucket.org/pypy/pypy/changeset/aa4ad227978a/

Log:	Catch and wrap the RPython-level OSError

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2219,13 +2219,20 @@
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
-    return space.newbool(rposix.get_status_flags(fd) & rposix.O_NONBLOCK == 0)
+    try:
+        flags = rposix.get_status_flags(fd)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    return space.newbool(flags & rposix.O_NONBLOCK == 0)
 
 @unwrap_spec(fd=c_int, blocking=int)
 def set_blocking(space, fd, blocking):
-    flags = rposix.get_status_flags(fd)
-    if blocking:
-        flags &= ~rposix.O_NONBLOCK
-    else:
-        flags |= rposix.O_NONBLOCK
-    rposix.set_status_flags(fd, flags)
+    try:
+        flags = rposix.get_status_flags(fd)
+        if blocking:
+            flags &= ~rposix.O_NONBLOCK
+        else:
+            flags |= rposix.O_NONBLOCK
+        rposix.set_status_flags(fd, flags)
+    except OSError as e:
+        raise wrap_oserror(space, e)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1148,6 +1148,11 @@
             assert posix.get_blocking(fd) is True
             posix.close(fd)
 
+        def test_blocking_error(self):
+            posix = self.posix
+            raises(OSError, posix.get_blocking, 1234567)
+            raises(OSError, posix.set_blocking, 1234567, True)
+
 
     def test_urandom(self):
         os = self.posix

From pypy.commits at gmail.com  Tue Nov 29 04:24:17 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 01:24:17 -0800 (PST)
Subject: [pypy-commit] pypy default: posix.stat_float_times(): the default
 is not None but -1
Message-ID: <583d4941.c6bdc20a.29d75.8fd1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88727:68764720cf0c
Date: 2016-11-29 10:23 +0100
http://bitbucket.org/pypy/pypy/changeset/68764720cf0c/

Log:	posix.stat_float_times(): the default is not None but -1

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -307,7 +307,8 @@
     def __init__(self, space):
         self.stat_float_times = True
 
-def stat_float_times(space, w_value=None):
+ at unwrap_spec(newval=int)
+def stat_float_times(space, newval=-1):
     """stat_float_times([newval]) -> oldval
 
 Determine whether os.[lf]stat represents time stamps as float objects.
@@ -317,10 +318,10 @@
 """
     state = space.fromcache(StatState)
 
-    if w_value is None:
+    if newval == -1:
         return space.wrap(state.stat_float_times)
     else:
-        state.stat_float_times = space.bool_w(w_value)
+        state.stat_float_times = (newval != 0)
 
 
 @unwrap_spec(fd=c_int)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -159,11 +159,14 @@
             st = posix.stat(path)
             assert isinstance(st.st_mtime, float)
             assert st[7] == int(st.st_atime)
+            assert posix.stat_float_times(-1) is True
 
             posix.stat_float_times(False)
             st = posix.stat(path)
             assert isinstance(st.st_mtime, (int, long))
             assert st[7] == st.st_atime
+            assert posix.stat_float_times(-1) is False
+
         finally:
             posix.stat_float_times(current)
 

From pypy.commits at gmail.com  Tue Nov 29 04:24:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 01:24:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <583d4943.0e0a1c0a.53868.8cd3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88728:6c1c69ada9bc
Date: 2016-11-29 10:23 +0100
http://bitbucket.org/pypy/pypy/changeset/6c1c69ada9bc/

Log:	hg merge default

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -484,7 +484,8 @@
     def __init__(self, space):
         self.stat_float_times = True
 
-def stat_float_times(space, w_value=None):
+ at unwrap_spec(newval=int)
+def stat_float_times(space, newval=-1):
     """stat_float_times([newval]) -> oldval
 
 Determine whether os.[lf]stat represents time stamps as float objects.
@@ -494,10 +495,10 @@
 """
     state = space.fromcache(StatState)
 
-    if w_value is None:
+    if newval == -1:
         return space.wrap(state.stat_float_times)
     else:
-        state.stat_float_times = space.bool_w(w_value)
+        state.stat_float_times = (newval != 0)
 
 
 @unwrap_spec(fd=c_int)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -159,11 +159,14 @@
             st = posix.stat(path)
             assert isinstance(st.st_mtime, float)
             assert st[7] == int(st.st_atime)
+            assert posix.stat_float_times(-1) is True
 
             posix.stat_float_times(False)
             st = posix.stat(path)
             assert isinstance(st.st_mtime, int)
             assert st[7] == st.st_atime
+            assert posix.stat_float_times(-1) is False
+
         finally:
             posix.stat_float_times(current)
 

From pypy.commits at gmail.com  Tue Nov 29 05:40:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 02:40:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Fix a corner case for weakref.__ne__()
Message-ID: <583d5b07.45f6c20a.f25f.84cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88729:ebe85c51ad48
Date: 2016-11-29 11:39 +0100
http://bitbucket.org/pypy/pypy/changeset/ebe85c51ad48/

Log:	Fix a corner case for weakref.__ne__()

diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -217,7 +217,7 @@
             return self.space.w_None
         return w_obj
 
-    def descr__eq__(self, space, w_ref2):
+    def compare(self, space, w_ref2, invert):
         if not isinstance(w_ref2, W_Weakref):
             return space.w_NotImplemented
         ref1 = self
@@ -225,11 +225,18 @@
         w_obj1 = ref1.dereference()
         w_obj2 = ref2.dereference()
         if w_obj1 is None or w_obj2 is None:
-            return space.is_(ref1, ref2)
-        return space.eq(w_obj1, w_obj2)
+            w_res = space.is_(ref1, ref2)
+        else:
+            w_res = space.eq(w_obj1, w_obj2)
+        if invert:
+            w_res = space.not_(w_res)
+        return w_res
+
+    def descr__eq__(self, space, w_ref2):
+        return self.compare(space, w_ref2, invert=False)
 
     def descr__ne__(self, space, w_ref2):
-        return space.not_(space.eq(self, w_ref2))
+        return self.compare(space, w_ref2, invert=True)
 
 def getlifeline(space, w_obj):
     lifeline = w_obj.getweakref()
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -150,6 +150,14 @@
         assert not (ref1 == [])
         assert ref1 != []
 
+    def test_ne(self):
+        import _weakref
+        class X(object):
+            pass
+        ref1 = _weakref.ref(X())
+        assert ref1.__eq__(X()) is NotImplemented
+        assert ref1.__ne__(X()) is NotImplemented
+
     def test_getweakrefs(self):
         import _weakref, gc
         class A(object):

From pypy.commits at gmail.com  Tue Nov 29 05:47:04 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 02:47:04 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: remove space.wrap in array
Message-ID: <583d5ca8.113cc20a.8d5cb.87d2@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88730:7b943d7552ef
Date: 2016-11-28 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/7b943d7552ef/

Log:	remove space.wrap in array

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -846,10 +846,16 @@
             if mytype.typecode in 'bBhHil':
                 item = rffi.cast(lltype.Signed, item)
                 return space.newint(item)
-            elif mytype.typecode == 'f':
+            if mytype.typecode in 'IL':
+                return space.newint(item)
+            elif mytype.typecode in 'fd':
                 item = float(item)
                 return space.newfloat(item)
-            return space.wrap(item) # YYY
+            elif mytype.typecode == 'c':
+                return space.newbytes(item)
+            elif mytype.typecode == 'u':
+                return space.newunicode(item)
+            assert 0, "unreachable"
 
         # interface
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -171,6 +171,7 @@
         W_SliceObject(w_start, w_end, w_step)
         return w_some_obj()
 
+    @specialize.argtype(1)
     def newint(self, x):
         return w_some_obj()
 

From pypy.commits at gmail.com  Tue Nov 29 05:47:08 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 02:47:08 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: fix the annoying wrap in _rawffi
Message-ID: <583d5cac.313ac20a.7a536.9b19@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88732:f8751e58e3a8
Date: 2016-11-29 10:39 +0100
http://bitbucket.org/pypy/pypy/changeset/f8751e58e3a8/

Log:	fix the annoying wrap in _rawffi

diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -441,13 +441,16 @@
             if c in TYPEMAP_PTR_LETTERS:
                 res = func(add_arg, argdesc, rffi.VOIDP)
                 return space.newint(rffi.cast(lltype.Unsigned, res))
+            if c in TYPEMAP_NUMBER_LETTERS:
+                return space.newint(func(add_arg, argdesc, ll_type))
             elif c == 'c':
                 return space.newbytes(func(add_arg, argdesc, ll_type))
+            elif c == 'u':
+                return space.newunicode(func(add_arg, argdesc, ll_type))
             elif c == 'f' or c == 'd' or c == 'g':
                 return space.newfloat(float(func(add_arg, argdesc, ll_type)))
             else:
-                # YYY hard
-                return space.wrap(func(add_arg, argdesc, ll_type))
+                assert 0, "unreachable"
     raise oefmt(space.w_TypeError, "cannot directly read value")
 
 NARROW_INTEGER_TYPES = 'cbhiBIH?'

From pypy.commits at gmail.com  Tue Nov 29 05:47:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 02:47:06 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: remove wrap in struct
Message-ID: <583d5caa.41a3c20a.7f663.9e45@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88731:10f44f1fe7f8
Date: 2016-11-28 16:20 +0100
http://bitbucket.org/pypy/pypy/changeset/10f44f1fe7f8/

Log:	remove wrap in struct

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -151,18 +151,27 @@
     @specialize.argtype(1)
     def appendobj(self, value):
         # CPython tries hard to return int objects whenever it can, but
-        # space.wrap returns a long if we pass a r_uint, r_ulonglong or
+        # space.newint returns a long if we pass a r_uint, r_ulonglong or
         # r_longlong. So, we need special care in those cases.
         is_unsigned = (isinstance(value, r_uint) or
                        isinstance(value, r_ulonglong))
-        if is_unsigned and value <= maxint:
-            w_value = self.space.wrap(intmask(value))
+        if is_unsigned:
+            if value <= maxint:
+                w_value = self.space.newint(intmask(value))
+            else:
+                w_value = self.space.newint(value)
         elif isinstance(value, r_longlong) and -maxint-1 <= value <= maxint:
-            w_value = self.space.wrap(intmask(value))
+            w_value = self.space.newint(intmask(value))
+        elif isinstance(value, int):
+            w_value = self.space.newint(value)
+        elif isinstance(value, float):
+            w_value = self.space.newfloat(value)
+        elif isinstance(value, str):
+            w_value = self.space.newbytes(value)
+        elif isinstance(value, unicode):
+            w_value = self.space.newunicode(value)
         else:
-            # generic type, just use space.wrap
-            w_value = self.space.wrap(value)
-        #
+            assert 0, "unreachable"
         self.result_w.append(w_value)
 
     def get_pos(self):

From pypy.commits at gmail.com  Tue Nov 29 05:47:15 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 02:47:15 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: remove wrap calls in the objspace
 implementation itself
Message-ID: <583d5cb3.07941c0a.76cb1.a9e0@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88733:7608447652cb
Date: 2016-11-29 10:51 +0100
http://bitbucket.org/pypy/pypy/changeset/7608447652cb/

Log:	remove wrap calls in the objspace implementation itself

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -786,10 +786,12 @@
         return self.is_true(self.contains(w_container, w_item))
 
     def setitem_str(self, w_obj, key, w_value):
-        return self.setitem(w_obj, self.wrap(key), w_value)
+        # key is a "text" (ie str in python2 and unicode in python3)
+        return self.setitem(w_obj, self.newtext(key), w_value)
 
     def finditem_str(self, w_obj, key):
-        return self.finditem(w_obj, self.wrap(key))
+        # key is a "text" (ie str in python2 and unicode in python3)
+        return self.finditem(w_obj, self.newtext(key))
 
     def finditem(self, w_obj, w_key):
         try:
@@ -832,11 +834,12 @@
         return w_s1
 
     def new_interned_str(self, s):
+        # returns a "text" (ie str in python2 and unicode in python3)
         if not we_are_translated():
             assert type(s) is str
         w_s1 = self.interned_strings.get(s)
         if w_s1 is None:
-            w_s1 = self.wrap(s)
+            w_s1 = self.newbytes(s)
             self.interned_strings.set(s, w_s1)
         return w_s1
 
@@ -1174,7 +1177,7 @@
         return w_res
 
     def call_method(self, w_obj, methname, *arg_w):
-        w_meth = self.getattr(w_obj, self.wrap(methname))
+        w_meth = self.getattr(w_obj, self.newbytes(methname))
         return self.call_function(w_meth, *arg_w)
 
     def raise_key_error(self, w_key):
@@ -1183,7 +1186,7 @@
 
     def lookup(self, w_obj, name):
         w_type = self.type(w_obj)
-        w_mro = self.getattr(w_type, self.wrap("__mro__"))
+        w_mro = self.getattr(w_type, self.newbytes("__mro__"))
         for w_supertype in self.fixedview(w_mro):
             w_value = w_supertype.getdictvalue(self, name)
             if w_value is not None:
@@ -1204,7 +1207,7 @@
             if self.is_oldstyle_instance(w_obj):
                 # ugly old style class special treatment, but well ...
                 try:
-                    self.getattr(w_obj, self.wrap("__call__"))
+                    self.getattr(w_obj, self.newbytes("__call__"))
                     return self.w_True
                 except OperationError as e:
                     if not e.match(self, self.w_AttributeError):
@@ -1216,7 +1219,7 @@
 
     def issequence_w(self, w_obj):
         if self.is_oldstyle_instance(w_obj):
-            return (self.findattr(w_obj, self.wrap('__getitem__')) is not None)
+            return (self.findattr(w_obj, self.newbytes('__getitem__')) is not None)
         flag = self.type(w_obj).flag_map_or_seq
         if flag == 'M':
             return False
@@ -1227,7 +1230,7 @@
 
     def ismapping_w(self, w_obj):
         if self.is_oldstyle_instance(w_obj):
-            return (self.findattr(w_obj, self.wrap('__getitem__')) is not None)
+            return (self.findattr(w_obj, self.newbytes('__getitem__')) is not None)
         flag = self.type(w_obj).flag_map_or_seq
         if flag == 'M':
             return True
@@ -1305,15 +1308,15 @@
                                          hidden_applevel=hidden_applevel)
         if not isinstance(statement, PyCode):
             raise TypeError('space.exec_(): expected a string, code or PyCode object')
-        w_key = self.wrap('__builtins__')
+        w_key = self.newbytes('__builtins__')
         if not self.contains_w(w_globals, w_key):
-            self.setitem(w_globals, w_key, self.wrap(self.builtin))
+            self.setitem(w_globals, w_key, self.builtin)
         return statement.exec_code(self, w_globals, w_locals)
 
     @specialize.arg(2)
     def appexec(self, posargs_w, source):
         """ return value from executing given source at applevel.
-            EXPERIMENTAL. The source must look like
+            The source must look like
                '''(x, y):
                        do_stuff...
                        return result
@@ -1417,7 +1420,7 @@
                 raise
             if not w_exception:
                 # w_index should be a long object, but can't be sure of that
-                if self.is_true(self.lt(w_index, self.wrap(0))):
+                if self.is_true(self.lt(w_index, self.newint(0))):
                     return -sys.maxint-1
                 else:
                     return sys.maxint
@@ -1780,7 +1783,7 @@
         if (not self.isinstance_w(w_fd, self.w_int) and
             not self.isinstance_w(w_fd, self.w_long)):
             try:
-                w_fileno = self.getattr(w_fd, self.wrap("fileno"))
+                w_fileno = self.getattr(w_fd, self.newbytes("fileno"))
             except OperationError as e:
                 if e.match(self, self.w_AttributeError):
                     raise oefmt(self.w_TypeError,
@@ -1805,7 +1808,7 @@
         return fd
 
     def warn(self, w_msg, w_warningcls, stacklevel=2):
-        self.appexec([w_msg, w_warningcls, self.wrap(stacklevel)],
+        self.appexec([w_msg, w_warningcls, self.newint(stacklevel)],
                      """(msg, warningcls, stacklevel):
             import _warnings
             _warnings.warn(msg, warningcls, stacklevel=stacklevel)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -536,7 +536,7 @@
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
             return (a, b, c)
-        w_indices = self.getattr(w_slice, self.wrap('indices'))
+        w_indices = self.getattr(w_slice, self.newbytes('indices'))
         w_tup = self.call_function(w_indices, w_length)
         l_w = self.unpackiterable(w_tup)
         if not len(l_w) == 3:
@@ -629,7 +629,7 @@
                 not w_obj.user_overridden_class):
             w_obj.setitem_str(key, w_value)
         else:
-            self.setitem(w_obj, self.wrap(key), w_value)
+            self.setitem(w_obj, self.newtext(key), w_value)
 
     def getindex_w(self, w_obj, w_exception, objdescr=None):
         if type(w_obj) is W_IntObject:

From pypy.commits at gmail.com  Tue Nov 29 07:01:49 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 29 Nov 2016 04:01:49 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: use _cffi_ssl to implement _hashlib
Message-ID: <583d6e2d.8a29c20a.df6a.b483@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88734:d9e5c51789d1
Date: 2016-11-29 12:23 +0100
http://bitbucket.org/pypy/pypy/changeset/d9e5c51789d1/

Log:	use _cffi_ssl to implement _hashlib

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
@@ -16,6 +16,13 @@
 
 typedef ... EVP_PKEY;
 typedef ... EVP_PKEY_CTX;
+typedef struct {
+    int type;
+    int alias;
+    const char *name;
+    const char *data;
+} OBJ_NAME;
+
 static const int EVP_PKEY_RSA;
 static const int EVP_PKEY_DSA;
 static const int EVP_PKEY_DH;
@@ -136,6 +143,7 @@
    without worrying about what OpenSSL we're running against. */
 EVP_MD_CTX *Cryptography_EVP_MD_CTX_new(void);
 void Cryptography_EVP_MD_CTX_free(EVP_MD_CTX *);
+void OBJ_NAME_do_all(int, void (*) (const OBJ_NAME *, void *), void *);
 """
 
 MACROS = """
@@ -156,6 +164,7 @@
 EC_KEY *EVP_PKEY_get1_EC_KEY(EVP_PKEY *);
 int EVP_PKEY_set1_EC_KEY(EVP_PKEY *, EC_KEY *);
 
+int EVP_MD_CTX_block_size(const EVP_MD_CTX *md);
 int EVP_CIPHER_CTX_block_size(const EVP_CIPHER_CTX *);
 int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *, int, int, void *);
 
@@ -167,6 +176,8 @@
 int EVP_PBE_scrypt(const char *, size_t, const unsigned char *, size_t,
                    uint64_t, uint64_t, uint64_t, uint64_t, unsigned char *,
                    size_t);
+
+#define OBJ_NAME_TYPE_MD_METH ...
 """
 
 CUSTOMIZATIONS = """
diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_hashlib/__init__.py
@@ -0,0 +1,172 @@
+import sys
+from threading import Lock
+from _pypy_openssl import ffi, lib
+from _cffi_ssl._stdssl.utility import (_str_to_ffi_buffer, _bytes_with_len,
+        _str_from_buf)
+
+def new(name, string=b''):
+    h = Hash(name)
+    h.update(string)
+    return h
+
+class Hash(object):
+
+    def __init__(self, name, copy_from=None):
+        self.ctx = ffi.NULL
+        self.name = name
+        digest_type = self.digest_type_by_name()
+        self.digest_size = lib.EVP_MD_size(digest_type)
+
+        # Allocate a lock for each HASH object.
+        # An optimization would be to not release the GIL on small requests,
+        # and use a custom lock only when needed.
+        self.lock = Lock()
+
+        ctx = lib.Cryptography_EVP_MD_CTX_new()
+        if ctx == ffi.NULL:
+            raise MemoryError
+        ctx = ffi.gc(ctx, lib.Cryptography_EVP_MD_CTX_free)
+
+        try:
+            if copy_from is not None:
+                # cpython uses EVP_MD_CTX_copy(...)
+                if not lib.EVP_MD_CTX_copy_ex(ctx, copy_from):
+                    raise ValueError
+            else:
+                # cpython uses EVP_DigestInit
+                lib.EVP_DigestInit_ex(ctx, digest_type, ffi.NULL)
+            self.ctx = ctx
+        except:
+            # no need to gc ctx! 
+            raise
+
+    def digest_type_by_name(self):
+        c_name = _str_to_ffi_buffer(self.name)
+        digest_type = lib.EVP_get_digestbyname(c_name)
+        if not digest_type:
+            raise ValueError("unknown hash function")
+        # TODO
+        return digest_type
+
+    def __repr__(self):
+        return "<%s HASH object at 0x%s>" % (self.name, id(self))
+
+    def update(self, string):
+        buf = ffi.from_buffer(string)
+        with self.lock:
+            # XXX try to not release the GIL for small requests
+            lib.EVP_DigestUpdate(self.ctx, buf, len(buf))
+
+    def copy(self):
+        """Return a copy of the hash object."""
+        with self.lock:
+            return Hash(self.name, copy_from=self.ctx)
+
+    def digest(self):
+        """Return the digest value as a string of binary data."""
+        return self._digest()
+
+    def hexdigest(self):
+        """Return the digest value as a string of hexadecimal digits."""
+        digest = self._digest()
+        hexdigits = '0123456789abcdef'
+        result = []
+        for c in digest:
+            result.append(hexdigits[(c >> 4) & 0xf])
+            result.append(hexdigits[ c       & 0xf])
+        return ''.join(result)
+
+    @property
+    def block_size(self):
+        return lib.EVP_MD_CTX_block_size(self.ctx)
+
+    def _digest(self):
+        ctx = lib.Cryptography_EVP_MD_CTX_new()
+        if ctx == ffi.NULL:
+            raise MemoryError
+        try:
+            with self.lock:
+                if not lib.EVP_MD_CTX_copy_ex(ctx, self.ctx):
+                    raise ValueError
+            digest_size = self.digest_size
+            buf = ffi.new("unsigned char[]", digest_size)
+            lib.EVP_DigestFinal_ex(ctx, buf, ffi.NULL)
+            return _bytes_with_len(buf, digest_size)
+        finally:
+            lib.Cryptography_EVP_MD_CTX_free(ctx)
+
+algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
+
+class NameFetcher:
+    def __init__(self):
+        self.meth_names = []
+        self.error = None
+
+
+def _fetch_names():
+    name_fetcher = NameFetcher()
+    handle = ffi.new_handle(name_fetcher)
+    lib.OBJ_NAME_do_all(lib.OBJ_NAME_TYPE_MD_METH, hash_name_mapper_callback, handle)
+    if name_fetcher.error:
+        raise name_fetcher.error
+    meth_names = name_fetcher.meth_names
+    name_fetcher.meth_names = None
+    return frozenset(meth_names)
+
+ at ffi.callback("void(OBJ_NAME*, void*)")
+def hash_name_mapper_callback(obj_name, userdata):
+    if not obj_name:
+        return
+    name_fetcher = ffi.from_handle(userdata)
+    # Ignore aliased names, they pollute the list and OpenSSL appears
+    # to have a its own definition of alias as the resulting list
+    # still contains duplicate and alternate names for several
+    # algorithms.
+    if obj_name.alias != 0:
+        return
+    name = _str_from_buf(obj_name.name)
+    name_fetcher.meth_names.append(name)
+
+openssl_md_meth_names = _fetch_names()
+del _fetch_names
+
+# shortcut functions
+def make_new_hash(name, funcname):
+    def new_hash(string=b''):
+        return new(name, string)
+    new_hash.__name__ = funcname
+    return new_hash
+
+for _name in algorithms:
+    _newname = 'openssl_%s' % (_name,)
+    globals()[_newname] = make_new_hash(_name, _newname)
+
+if hasattr(lib, 'PKCS5_PBKDF2_HMAC'):
+    #@unwrap_spec(name=str, password='bytes', salt='bytes', iterations=int,
+    #             w_dklen=WrappedDefault(None))
+    def pbkdf2_hmac(name, password, salt, iterations, dklen=None):
+        if not isinstance(name, str):
+            raise TypeError("expected 'str' for name, but got %s" % type(name))
+        c_name = _str_to_ffi_buffer(name)
+        digest = lib.EVP_get_digestbyname(c_name)
+        if digest == ffi.NULL:
+            raise ValueError("unsupported hash type")
+        if dklen is None:
+            dklen = lib.EVP_MD_size(digest)
+        if dklen < 1:
+            raise ValueError("key length must be greater than 0.")
+        if dklen >= sys.maxsize:
+            raise OverflowError("key length is too great.")
+        if iterations < 1:
+            raise ValueError("iteration value must be greater than 0.")
+        if iterations >= sys.maxsize:
+            raise OverflowError("iteration value is too great.")
+        buf = ffi.new("unsigned char[]", dklen)
+        c_password = ffi.from_buffer(bytes(password))
+        c_salt = ffi.from_buffer(bytes(salt))
+        r = lib.PKCS5_PBKDF2_HMAC(c_password, len(c_password),
+                ffi.cast("unsigned char*",c_salt), len(c_salt),
+                iterations, digest, dklen, buf)
+        if r == 0:
+            raise ValueError
+        return _bytes_with_len(buf, dklen)

From pypy.commits at gmail.com  Tue Nov 29 07:01:51 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 29 Nov 2016 04:01:51 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: update cryptography cffi backend
Message-ID: <583d6e2f.c9b3c20a.40d2f.abe2@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88735:b3977f174074
Date: 2016-11-29 13:01 +0100
http://bitbucket.org/pypy/pypy/changeset/b3977f174074/

Log:	update cryptography cffi backend

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py
@@ -53,6 +53,7 @@
 const char *OpenSSL_version(int);
 
 /* this is a macro in 1.1.0 */
+void *OPENSSL_malloc(size_t);
 void OPENSSL_free(void *);
 
 /* This was removed in 1.1.0 */
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py
@@ -16,13 +16,6 @@
 
 typedef ... EVP_PKEY;
 typedef ... EVP_PKEY_CTX;
-typedef struct {
-    int type;
-    int alias;
-    const char *name;
-    const char *data;
-} OBJ_NAME;
-
 static const int EVP_PKEY_RSA;
 static const int EVP_PKEY_DSA;
 static const int EVP_PKEY_DH;
@@ -32,6 +25,13 @@
 static const int EVP_CTRL_GCM_GET_TAG;
 static const int EVP_CTRL_GCM_SET_TAG;
 
+typedef struct {
+    int type;
+    int alias;
+    const char *name;
+    const char *data;
+} OBJ_NAME;
+
 static const int Cryptography_HAS_GCM;
 static const int Cryptography_HAS_PBKDF2_HMAC;
 static const int Cryptography_HAS_PKEY_CTX;
@@ -176,7 +176,6 @@
 int EVP_PBE_scrypt(const char *, size_t, const unsigned char *, size_t,
                    uint64_t, uint64_t, uint64_t, uint64_t, unsigned char *,
                    size_t);
-
 #define OBJ_NAME_TYPE_MD_METH ...
 """
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -26,7 +26,6 @@
 static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE;
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS;
 static const long Cryptography_HAS_NPN_NEGOTIATED;
-static const long Cryptography_OPENSSL_NO_TLSEXT;
 
 /* Internally invented symbol to tell us if SNI is supported */
 static const long Cryptography_HAS_TLSEXT_HOSTNAME;
@@ -138,6 +137,7 @@
 typedef ... SSL_CTX;
 
 typedef ... SSL_SESSION;
+
 typedef ... SSL;
 
 static const long TLSEXT_NAMETYPE_host_name;
@@ -434,7 +434,6 @@
 long SSL_CTX_sess_misses(SSL_CTX *);
 long SSL_CTX_sess_timeouts(SSL_CTX *);
 long SSL_CTX_sess_cache_full(SSL_CTX *);
-
 """
 
 CUSTOMIZATIONS = """
@@ -689,12 +688,6 @@
 
 static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1;
 
-#ifdef OPENSSL_NO_TLSEXT
-static const long Cryptography_OPENSSL_NO_TLSEXT = 1;
-#else
-static const long Cryptography_OPENSSL_NO_TLSEXT = 0;
-#endif
-
 /* in OpenSSL 1.1.0 the SSL_ST values were renamed to TLS_ST and several were
    removed */
 #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 || defined(LIBRESSL_VERSION_NUMBER)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
@@ -140,6 +140,7 @@
 int X509_STORE_set_flags(X509_STORE *, unsigned long);
 void X509_STORE_free(X509_STORE *);
 
+
 /* X509_STORE_CTX */
 X509_STORE_CTX *X509_STORE_CTX_new(void);
 void X509_STORE_CTX_cleanup(X509_STORE_CTX *);
@@ -201,7 +202,7 @@
 
 int sk_X509_OBJECT_num(Cryptography_STACK_OF_X509_OBJECT *);
 X509_OBJECT *sk_X509_OBJECT_value(Cryptography_STACK_OF_X509_OBJECT *, int);
-X509_VERIFY_PARAM * X509_STORE_get0_param(X509_STORE *);
+X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *);
 Cryptography_STACK_OF_X509_OBJECT *X509_STORE_get0_objects(X509_STORE *);
 X509 *X509_OBJECT_get0_X509(X509_OBJECT *);
 int X509_OBJECT_get_type(const X509_OBJECT *);

From pypy.commits at gmail.com  Tue Nov 29 08:25:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 05:25:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Add rgc.may_ignore_finalizer(): an
 optimization hint that makes the
Message-ID: <583d81b3.05371c0a.2315c.eb73@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88736:a42d0357c952
Date: 2016-11-29 14:22 +0100
http://bitbucket.org/pypy/pypy/changeset/a42d0357c952/

Log:	Add rgc.may_ignore_finalizer(): an optimization hint that makes the
	GC stop tracking the object---its finalizer(s) won't be called,
	then.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -155,7 +155,10 @@
 # 'old_objects_pointing_to_pinned' and doesn't have to be added again.
 GCFLAG_PINNED_OBJECT_PARENT_KNOWN = GCFLAG_PINNED
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 10    # the first unused bit
+# record that ignore_finalizer() has been called
+GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 11    # the first unused bit
 
 
 # States for the incremental GC
@@ -1672,7 +1675,7 @@
             self.rrc_minor_collection_trace()
         #
         # visit the "probably young" objects with finalizers.  They
-        # always all survive.
+        # all survive, except if IGNORE_FINALIZER is set.
         if self.probably_young_objects_with_finalizers.non_empty():
             self.deal_with_young_objects_with_finalizers()
         #
@@ -2675,6 +2678,8 @@
         while self.probably_young_objects_with_finalizers.non_empty():
             obj = self.probably_young_objects_with_finalizers.popleft()
             fq_nr = self.probably_young_objects_with_finalizers.popleft()
+            if self.header(obj).tid & GCFLAG_IGNORE_FINALIZER:
+                continue
             self.singleaddr.address[0] = obj
             self._trace_drag_out1(self.singleaddr)
             obj = self.singleaddr.address[0]
@@ -2697,6 +2702,8 @@
             fq_nr = self.old_objects_with_finalizers.popleft()
             ll_assert(self._finalization_state(x) != 1,
                       "bad finalization state 1")
+            if self.header(x).tid & GCFLAG_IGNORE_FINALIZER:
+                continue
             if self.header(x).tid & GCFLAG_VISITED:
                 new_with_finalizer.append(x)
                 new_with_finalizer.append(fq_nr)
@@ -2787,6 +2794,9 @@
         self.objects_to_trace.append(obj)
         self.visit_all_objects()
 
+    def ignore_finalizer(self, obj):
+        self.header(obj).tid |= GCFLAG_IGNORE_FINALIZER
+
 
     # ----------
     # Weakrefs
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -545,6 +545,12 @@
                                              s_gcref],
                                             annmodel.s_None)
 
+        self.ignore_finalizer_ptr = None
+        if hasattr(GCClass, 'ignore_finalizer'):
+            self.ignore_finalizer_ptr = getfn(GCClass.ignore_finalizer,
+                                              [s_gc, SomeAddress()],
+                                              annmodel.s_None)
+
     def create_custom_trace_funcs(self, gc, rtyper):
         custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
         rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1572,6 +1578,13 @@
         hop.genop("cast_adr_to_ptr", [v_adr],
                   resultvar = hop.spaceop.result)
 
+    def gct_gc_ignore_finalizer(self, hop):
+        if self.ignore_finalizer_ptr is not None:
+            v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+                              resulttype=llmemory.Address)
+            hop.genop("direct_call", [self.ignore_finalizer_ptr,
+                                      self.c_const_gc, v_adr])
+
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -527,6 +527,12 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
+def may_ignore_finalizer(obj):
+    """Optimization hint: says that it is valid for any finalizer
+    for 'obj' to be ignored, depending on the GC."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_ignore_finalizer(lltype.Void, obj)
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -486,6 +486,7 @@
     'gc_add_memory_pressure': LLOp(),
     'gc_fq_next_dead'     : LLOp(),
     'gc_fq_register'      : LLOp(),
+    'gc_ignore_finalizer' : LLOp(canrun=True),
 
     'gc_rawrefcount_init':              LLOp(),
     'gc_rawrefcount_create_link_pypy':  LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -736,6 +736,9 @@
     assert isinstance(x, bool)
     return x
 
+def op_gc_ignore_finalizer(obj):
+    pass
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h
--- a/rpython/translator/c/src/mem.h
+++ b/rpython/translator/c/src/mem.h
@@ -152,6 +152,7 @@
 #define OP_GC_IS_RPY_INSTANCE(x, r)      r = 0
 #define OP_GC_DUMP_RPY_HEAP(fd, r)       r = 0
 #define OP_GC_SET_EXTRA_THRESHOLD(x, r)  /* nothing */
+#define OP_GC_IGNORE_FINALIZER(x, r)     /* nothing */
 
 /****************************/
 /* The "asmgcc" root finder */
diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py
--- a/rpython/translator/c/test/test_boehm.py
+++ b/rpython/translator/c/test/test_boehm.py
@@ -409,7 +409,9 @@
         #
         def fn():
             for i in range(1000):
-                fq.register_finalizer(A(i))
+                x = A(i)
+                fq.register_finalizer(x)
+                rgc.may_ignore_finalizer(x)   # this is ignored with Boehm
             rgc.collect()
             rgc.collect()
             if glob.triggered == 0:
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1705,6 +1705,38 @@
             res = self.run("limited_memory_linux", -1, runner=myrunner)
             assert res == 42
 
+    def define_ignore_finalizer(cls):
+        class X(object):
+            pass
+        class FQ(rgc.FinalizerQueue):
+            Class = X
+            def finalizer_trigger(self):
+                pass
+        queue = FQ()
+        def g():
+            x1 = X()
+            x2 = X()
+            queue.register_finalizer(x1)
+            queue.register_finalizer(x2)
+            rgc.may_ignore_finalizer(x1)
+        g._dont_inline_ = True
+        def f():
+            g()
+            rgc.collect()
+            seen = 0
+            while True:
+                obj = queue.next_dead()
+                if obj is None:
+                    break
+                seen += 1
+            return seen
+        assert f() == 2    # untranslated: may_ignore_finalizer() is ignored
+        return f
+
+    def test_ignore_finalizer(self):
+        res = self.run("ignore_finalizer")
+        assert res == 1    # translated: x1 is removed from the list
+
 
 # ____________________________________________________________________
 

From pypy.commits at gmail.com  Tue Nov 29 08:31:56 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 05:31:56 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: make space.wrap as @not_rpython
Message-ID: <583d834c.ce841c0a.b0b67.ef47@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88737:5f267995aab4
Date: 2016-11-29 14:31 +0100
http://bitbucket.org/pypy/pypy/changeset/5f267995aab4/

Log:	make space.wrap as @not_rpython

diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py
--- a/pypy/module/_file/interp_stream.py
+++ b/pypy/module/_file/interp_stream.py
@@ -120,7 +120,7 @@
         finally:
             if acquired:
                 self.release_lock()
-        return space.wrap(result) # YYY
+        return space.wrap(result)
     %(name)s.unwrap_spec = [W_Stream, ObjSpace] + argtypes
     """ % locals()).compile() in globals()
 
diff --git a/pypy/module/cppyy/__init__.py b/pypy/module/cppyy/__init__.py
--- a/pypy/module/cppyy/__init__.py
+++ b/pypy/module/cppyy/__init__.py
@@ -41,4 +41,4 @@
         from pypy.module.cppyy import capi
         capi.verify_backend(space)      # may raise ImportError
 
-        space.call_method(space.wrap(self), '_init_pythonify')
+        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -758,4 +758,5 @@
     Unicode strings.  CRLF is considered to be one line break.  If
     keepend is 0, the Line break characters are not included in the
     resulting strings."""
-    return space.call_method(w_str, "splitlines", space.newbool(bool(keepend)))
+    w_keepend = space.newbool(bool(rffi.cast(lltype.Signed, keepend)))
+    return space.call_method(w_str, "splitlines", w_keepend)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -859,7 +859,7 @@
             alignment = -1
         format = dtype_from_spec(space, obj[0], alignment=alignment)
         if len(obj) > 2:
-            title = space.wrap(obj[2])
+            title = obj[2]
         else:
             title = space.w_None
         allfields.append((fname_w, format, num, title))
@@ -873,7 +873,7 @@
     else:
         alignment = -1
     for i in range(len(names)):
-        aslist.append(space.newtuple([space.wrap(names[i]), space.wrap(formats[i])]))
+        aslist.append(space.newtuple([names[i], formats[i]]))
     return dtype_from_list(space, space.newlist(aslist), False, alignment, offsets=offsets)
     
 def dtype_from_dict(space, w_dict, alignment):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -8,7 +8,7 @@
 from rpython.rlib.objectmodel import instantiate, specialize, is_annotation_constant
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rarithmetic import base_int, widen, is_valid_int
-from rpython.rlib.objectmodel import import_from_mixin, enforceargs
+from rpython.rlib.objectmodel import import_from_mixin, enforceargs, not_rpython
 from rpython.rlib import jit
 
 # Object imports
@@ -39,8 +39,9 @@
     library in Restricted Python."""
     import_from_mixin(DescrOperation)
 
+    @not_rpython
     def initialize(self):
-        """NOT_RPYTHON: only for initializing the space
+        """only for initializing the space
 
         Setup all the object types and implementations.
         """
@@ -130,15 +131,11 @@
         assert typedef is not None
         return self.fromcache(TypeCache).getorbuild(typedef)
 
-    @specialize.argtype(1)
+    @not_rpython # only for tests
     def wrap(self, x):
-        "Wraps the Python value 'x' into one of the wrapper classes."
-        # You might notice that this function is rather conspicuously
-        # not RPython.  We can get away with this because the function
-        # is specialized (see after the function body).  Also worth
-        # noting is that the isinstance's involving integer types
-        # behave rather differently to how you might expect during
-        # annotation (see pypy/annotation/builtin.py)
+        """ Wraps the Python value 'x' into one of the wrapper classes. This
+        should only be used for tests, in real code you need to use the
+        explicit new* methods."""
         if x is None:
             return self.w_None
         if isinstance(x, OperationError):
@@ -161,11 +158,6 @@
             return w_result
         if isinstance(x, base_int):
             return self.newint(x)
-        return self._wrap_not_rpython(x)
-
-    def _wrap_not_rpython(self, x):
-        "NOT_RPYTHON"
-        # _____ this code is here to support testing only _____
 
         # we might get there in non-translated versions if 'x' is
         # a long that fits the correct range.
@@ -214,15 +206,15 @@
             self.wrap("refusing to wrap cpython value %r" % (x,))
         )
 
+    @not_rpython
     def wrap_exception_cls(self, x):
-        """NOT_RPYTHON"""
         if hasattr(self, 'w_' + x.__name__):
             w_result = getattr(self, 'w_' + x.__name__)
             return w_result
         return None
 
+    @not_rpython
     def wraplong(self, x):
-        "NOT_RPYTHON"
         if self.config.objspace.std.withsmalllong:
             from rpython.rlib.rarithmetic import r_longlong
             try:
@@ -235,8 +227,8 @@
                 return W_SmallLongObject(rx)
         return W_LongObject.fromlong(x)
 
+    @not_rpython
     def unwrap(self, w_obj):
-        """NOT_RPYTHON"""
         # _____ this code is here to support testing only _____
         if isinstance(w_obj, W_Root):
             return w_obj.unwrap(self)

From pypy.commits at gmail.com  Tue Nov 29 09:36:29 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 29 Nov 2016 06:36:29 -0800 (PST)
Subject: [pypy-commit] pypy py3.5-ssl: merge py3.5
Message-ID: <583d926d.8c1f1c0a.30a1a.037b@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5-ssl
Changeset: r88738:709cadb7f491
Date: 2016-11-29 13:02 +0100
http://bitbucket.org/pypy/pypy/changeset/709cadb7f491/

Log:	merge py3.5

diff too long, truncating to 2000 out of 2056 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -74,6 +74,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
@@ -109,6 +122,12 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
     if args:
         vals = []
         for name in args:
diff --git a/lib-python/3/_weakrefset.py b/lib-python/3/_weakrefset.py
--- a/lib-python/3/_weakrefset.py
+++ b/lib-python/3/_weakrefset.py
@@ -65,7 +65,14 @@
                     yield item
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, item):
         try:
diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -510,6 +510,7 @@
             fut.add_done_callback(lambda *args: self.loop.stop())
             self.loop.run_forever()
             fut = None # Trigger Future.__del__ or futures._TracebackLogger
+            support.gc_collect()
             if PY34:
                 # Future.__del__ in Python 3.4 logs error with
                 # an actual exception context
diff --git a/lib-python/3/test/test_asyncio/test_futures.py b/lib-python/3/test/test_asyncio/test_futures.py
--- a/lib-python/3/test/test_asyncio/test_futures.py
+++ b/lib-python/3/test/test_asyncio/test_futures.py
@@ -238,6 +238,7 @@
         fut.set_exception(RuntimeError('boom'))
         del fut
         test_utils.run_briefly(self.loop)
+        support.gc_collect()
         self.assertTrue(m_log.error.called)
 
     @mock.patch('asyncio.base_events.logger')
diff --git a/lib-python/3/test/test_complex.py b/lib-python/3/test/test_complex.py
--- a/lib-python/3/test/test_complex.py
+++ b/lib-python/3/test/test_complex.py
@@ -310,7 +310,7 @@
         self.assertRaises(TypeError, float, 5+3j)
         self.assertRaises(ValueError, complex, "")
         self.assertRaises(TypeError, complex, None)
-        self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None)
+        self.assertRaisesRegex(TypeError, " 'NoneType'", complex, None)
         self.assertRaises(ValueError, complex, "\0")
         self.assertRaises(ValueError, complex, "3\09")
         self.assertRaises(TypeError, complex, "1", "2")
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -1013,6 +1013,7 @@
             self.assertNotEqual(wr(), None)
         else:
             self.fail("RecursionError not raised")
+        gc_collect()
         self.assertEqual(wr(), None)
 
     def test_errno_ENOTDIR(self):
diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py
--- a/lib-python/3/test/test_sys.py
+++ b/lib-python/3/test/test_sys.py
@@ -811,7 +811,12 @@
             ref = AtExit()
         """
         rc, stdout, stderr = assert_python_ok('-c', code)
-        self.assertEqual(stdout.rstrip(), b'True')
+        if test.support.check_impl_detail(cpython=True):
+            self.assertEqual(stdout.rstrip(), b'True')
+        else:
+            # the __del__ method may or may not have been called
+            # in other Python implementations
+            self.assertIn(stdout.rstrip(), {b'True', b''})
 
 
 @test.support.cpython_only
diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py
--- a/lib-python/3/test/test_weakref.py
+++ b/lib-python/3/test/test_weakref.py
@@ -1103,7 +1103,7 @@
         # Keep an iterator alive
         it = dct.items()
         try:
-            next(it)
+            print(next(it))
         except StopIteration:
             pass
         del items
@@ -1112,7 +1112,10 @@
         del it
         gc.collect()
         gc.collect()
+        print(list(dct.items()))
         n2 = len(dct)
+        print(len(dct))
+        print(weakref)
         # one item may be kept alive inside the iterator
         self.assertIn(n1, (0, 1))
         self.assertEqual(n2, 0)
diff --git a/lib-python/3/unittest/test/test_assertions.py b/lib-python/3/unittest/test/test_assertions.py
--- a/lib-python/3/unittest/test/test_assertions.py
+++ b/lib-python/3/unittest/test/test_assertions.py
@@ -2,6 +2,7 @@
 import warnings
 import weakref
 import unittest
+from test.support import gc_collect
 from itertools import product
 
 
@@ -124,8 +125,10 @@
                     self.foo()
 
         Foo("test_functional").run()
+        gc_collect()
         self.assertIsNone(wr())
         Foo("test_with").run()
+        gc_collect()
         self.assertIsNone(wr())
 
     def testAssertNotRegex(self):
diff --git a/lib-python/3/unittest/test/test_case.py b/lib-python/3/unittest/test/test_case.py
--- a/lib-python/3/unittest/test/test_case.py
+++ b/lib-python/3/unittest/test/test_case.py
@@ -18,7 +18,7 @@
     TestEquality, TestHashing, LoggingResult, LegacyLoggingResult,
     ResultWithNoStartTestRunStopTestRun
 )
-from test.support import captured_stderr
+from test.support import captured_stderr, gc_collect
 
 
 log_foo = logging.getLogger('foo')
@@ -1737,6 +1737,7 @@
         for method_name in ('test1', 'test2'):
             testcase = TestCase(method_name)
             testcase.run()
+            gc_collect()
             self.assertEqual(MyException.ninstance, 0)
 
 
diff --git a/lib-python/3/weakref.py b/lib-python/3/weakref.py
--- a/lib-python/3/weakref.py
+++ b/lib-python/3/weakref.py
@@ -147,7 +147,14 @@
         del self.data[key]
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data.values():
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, key):
         try:
@@ -376,11 +383,18 @@
         return self.data[ref(key)]
 
     def __len__(self):
-        if self._dirty_len and self._pending_removals:
-            # self._pending_removals may still contain keys which were
-            # explicitly removed, we have to scrub them (see issue #21173).
-            self._scrub_removals()
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        if self._dirty_len and self._pending_removals:
+#            # self._pending_removals may still contain keys which were
+#            # explicitly removed, we have to scrub them (see issue #21173).
+#            self._scrub_removals()
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __repr__(self):
         return "<%s at %#x>" % (self.__class__.__name__, id(self))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,7 @@
 
 Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
 call this function" returning a result.
+
+.. branch: desc-specialize
+
+Refactor FunctionDesc.specialize() and related code (RPython annotator).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -302,6 +302,12 @@
         elif config.objspace.usemodules.pypyjit:
             config.translation.jit = True
 
+        if config.objspace.usemodules.cpyext:
+            if config.translation.gc != 'incminimark':
+                raise Exception("The 'cpyext' module requires the 'incminimark'"
+                                " GC.  You need either 'targetpypystandalone.py"
+                                " --withoutmod-cpyext' or '--gc=incminimark'")
+
         config.translating = True
 
         import translate
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -595,7 +595,7 @@
     # set up the Ctrl-C => KeyboardInterrupt signal handler, if the
     # signal module is available
     try:
-        import signal
+        import _signal as signal
     except ImportError:
         pass
     else:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -559,8 +559,9 @@
                                           space.wrap(msg))
         return OperationError(exc, w_error)
 
+ at specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
-                  w_exception_class=None):
+                  w_exception_class=None, w_filename2=None):
     assert isinstance(e, OSError)
 
     if _WINDOWS and isinstance(e, WindowsError):
@@ -580,25 +581,31 @@
     else:
         exc = w_exception_class
     if w_filename is not None:
-        w_error = space.call_function(exc, space.wrap(errno),
-                                      space.wrap(msg), w_filename)
+        if w_filename2 is not None:
+            w_error = space.call_function(exc, space.wrap(errno),
+                                          space.wrap(msg), w_filename,
+                                          space.w_None, w_filename2)
+        else:
+            w_error = space.call_function(exc, space.wrap(errno),
+                                          space.wrap(msg), w_filename)
     else:
         w_error = space.call_function(exc, space.wrap(errno),
                                       space.wrap(msg))
     return OperationError(exc, w_error)
-wrap_oserror2._annspecialcase_ = 'specialize:arg(3)'
 
+ at specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
-                 w_exception_class=None):
+                 w_exception_class=None, filename2=None):
+    w_filename = None
+    w_filename2 = None
     if filename is not None:
-        return wrap_oserror2(space, e, space.wrap(filename),
-                             exception_name=exception_name,
-                             w_exception_class=w_exception_class)
-    else:
-        return wrap_oserror2(space, e, None,
-                             exception_name=exception_name,
-                             w_exception_class=w_exception_class)
-wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
+        w_filename = space.wrap(filename)
+        if filename2 is not None:
+            w_filename2 = space.wrap(filename2)
+    return wrap_oserror2(space, e, w_filename,
+                         exception_name=exception_name,
+                         w_exception_class=w_exception_class,
+                         w_filename2=w_filename2)
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -20,7 +20,8 @@
         self.running = False
         self._name = name           # may be null, use get_name()
         self._qualname = qualname   # may be null, use get_qualname()
-        if self.pycode.co_flags & CO_YIELD_INSIDE_TRY:
+        if (isinstance(self, Coroutine)    # XXX would be cool not to need this
+                or self.pycode.co_flags & CO_YIELD_INSIDE_TRY):
             self.register_finalizer(self.space)
         self.saved_operr = None
 
@@ -385,7 +386,7 @@
            self.frame.last_instr == -1:
             space = self.space
             msg = u"coroutine '%s' was never awaited" % self.get_qualname()
-            space.warn(space.w_RuntimeWarning, space.wrap(msg))
+            space.warn(space.wrap(msg), space.w_RuntimeWarning)
         GeneratorOrCoroutine._finalize_(self)
 
 
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -160,3 +160,20 @@
         else:
             assert False, "should have raised"
         """
+
+    def test_runtime_warning(self): """
+        import gc, warnings
+        async def foobaz():
+            pass
+        with warnings.catch_warnings(record=True) as l:
+            foobaz()
+            gc.collect()
+            gc.collect()
+            gc.collect()
+
+        assert len(l) == 1, repr(l)
+        w = l[0].message
+        assert isinstance(w, RuntimeWarning)
+        assert str(w).startswith("coroutine ")
+        assert str(w).endswith("foobaz' was never awaited")
+        """
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
@@ -7,7 +8,6 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.sliceobject import unwrap_start_stop
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -124,13 +124,13 @@
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         if can_be_None and w_obj is None:
             return None
         if not isinstance(w_obj, RequiredClass):
             raise TypeError
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def getarg_w(self, code, w_obj):    # for retrieving buffers
         return FakeBuffer(w_obj)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -35,8 +35,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -1622,9 +1620,8 @@
     miniglobals = {'__name__':    __name__, # for module name propagation
                    }
     exec source.compile() in miniglobals
-    call_external_function = miniglobals['cpy_call_external']
+    call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
     call_external_function._dont_inline_ = True
-    call_external_function._annspecialcase_ = 'specialize:ll'
     call_external_function._gctransformer_hint_close_stack_ = True
     # don't inline, as a hack to guarantee that no GC pointer is alive
     # anywhere in call_external_function
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,7 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import fatalerror
 
 
 #________________________________________________________
@@ -192,6 +193,8 @@
     rawrefcount.create_link_pypy(w_obj, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 def from_ref(space, ref):
     """
     Finds the interpreter object corresponding to the given reference.  If the
@@ -202,7 +205,23 @@
         return None
     w_obj = rawrefcount.to_obj(W_Root, ref)
     if w_obj is not None:
-        return w_obj
+        if w_obj is not w_marker_deallocating:
+            return w_obj
+        fatalerror(
+            "*** Invalid usage of a dying CPython object ***\n"
+            "\n"
+            "cpyext, the emulation layer, detected that while it is calling\n"
+            "an object's tp_dealloc, the C code calls back a function that\n"
+            "tries to recreate the PyPy version of the object.  Usually it\n"
+            "means that tp_dealloc calls some general PyXxx() API.  It is\n"
+            "a dangerous and potentially buggy thing to do: even in CPython\n"
+            "the PyXxx() function could, in theory, cause a reference to the\n"
+            "object to be taken and stored somewhere, for an amount of time\n"
+            "exceeding tp_dealloc itself.  Afterwards, the object will be\n"
+            "freed, making that reference point to garbage.\n"
+            ">>> PyPy could contain some workaround to still work if\n"
+            "you are lucky, but it is not done so far; better fix the bug in\n"
+            "the CPython extension.")
 
     # This reference is not yet a real interpreter object.
     # Realize it.
@@ -233,7 +252,8 @@
 INTERPLEVEL_API['as_pyobj'] = as_pyobj
 
 def pyobj_has_w_obj(pyobj):
-    return rawrefcount.to_obj(W_Root, pyobj) is not None
+    w_obj = rawrefcount.to_obj(W_Root, pyobj)
+    return w_obj is not None and w_obj is not w_marker_deallocating
 INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj)
 
 
@@ -335,6 +355,7 @@
     pto = obj.c_ob_type
     #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \
     #      "'s type which is", rffi.charp2str(pto.c_tp_name)
+    rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -446,6 +446,7 @@
         self.w_errno = space.w_None
         self.w_strerror = space.w_None
         self.w_filename = space.w_None
+        self.w_filename2 = space.w_None
         self.written = -1  # only for BlockingIOError.
         W_BaseException.__init__(self, space)
 
@@ -471,15 +472,17 @@
 
     @staticmethod
     def _parse_init_args(space, args_w):
-        if 2 <= len(args_w) <= 3:
+        if 2 <= len(args_w) <= 5:
             w_errno = args_w[0]
             w_strerror = args_w[1]
-            if len(args_w) == 3:
+            w_filename = None
+            w_filename2 = None
+            if len(args_w) > 2:
                 w_filename = args_w[2]
-            else:
-                w_filename = None
-            return w_errno, w_strerror, w_filename
-        return None, None, None
+                if len(args_w) > 4:
+                    w_filename2 = args_w[4]
+            return w_errno, w_strerror, w_filename, w_filename2
+        return None, None, None, None
 
     @staticmethod
     def descr_new(space, w_subtype, __args__):
@@ -487,11 +490,12 @@
         w_errno = None
         w_strerror = None
         w_filename = None
+        w_filename2 = None
         if not W_OSError._use_init(space, w_subtype):
             if kwds_w:
                 raise oefmt(space.w_TypeError,
                             "OSError does not take keyword arguments")
-            (w_errno, w_strerror, w_filename,
+            (w_errno, w_strerror, w_filename, w_filename2
              ) = W_OSError._parse_init_args(space, args_w)
         if (not space.is_none(w_errno) and
             space.is_w(w_subtype, space.gettypeobject(W_OSError.typedef))):
@@ -509,7 +513,8 @@
         exc = space.allocate_instance(W_OSError, w_subtype)
         W_OSError.__init__(exc, space)
         if not W_OSError._use_init(space, w_subtype):
-            exc._init_error(space, args_w, w_errno, w_strerror, w_filename)
+            exc._init_error(space, args_w, w_errno, w_strerror, w_filename,
+                            w_filename2)
         return space.wrap(exc)
 
     def descr_init(self, space, __args__):
@@ -520,11 +525,13 @@
         if kwds_w:
             raise oefmt(space.w_TypeError,
                         "OSError does not take keyword arguments")
-        (w_errno, w_strerror, w_filename
+        (w_errno, w_strerror, w_filename, w_filename2
          ) = W_OSError._parse_init_args(space, args_w)
-        self._init_error(space, args_w, w_errno, w_strerror, w_filename)
+        self._init_error(space, args_w, w_errno, w_strerror, w_filename,
+                         w_filename2)
 
-    def _init_error(self, space, args_w, w_errno, w_strerror, w_filename):
+    def _init_error(self, space, args_w, w_errno, w_strerror, w_filename,
+                    w_filename2):
         W_BaseException.descr_init(self, space, args_w)
         if w_errno:
             self.w_errno = w_errno
@@ -540,17 +547,21 @@
                     self.w_filename = w_filename
             else:
                 self.w_filename = w_filename
+                if not space.is_none(w_filename2):
+                    self.w_filename2 = w_filename2
                 # filename is removed from the args tuple (for compatibility
                 # purposes, see test_exceptions.py)
                 self.args_w = [w_errno, w_strerror]
 
     # since we rebind args_w, we need special reduce, grump
     def descr_reduce(self, space):
+        extra = []
         if not space.is_w(self.w_filename, space.w_None):
-            lst = [self.getclass(space), space.newtuple(
-                self.args_w + [self.w_filename])]
-        else:
-            lst = [self.getclass(space), space.newtuple(self.args_w)]
+            extra.append(self.w_filename)
+            if not space.is_w(self.w_filename2, space.w_None):
+                extra.append(space.w_None)
+                extra.append(self.w_filename2)
+        lst = [self.getclass(space), space.newtuple(self.args_w + extra)]
         if self.w_dict is not None and space.is_true(self.w_dict):
             lst = lst + [self.w_dict]
         return space.newtuple(lst)
@@ -561,6 +572,12 @@
             errno = space.unicode_w(space.str(self.w_errno))
             strerror = space.unicode_w(space.str(self.w_strerror))
             if not space.is_w(self.w_filename, space.w_None):
+                if not space.is_w(self.w_filename2, space.w_None):
+                    return space.wrap(u"[Errno %s] %s: %s -> %s" % (
+                        errno,
+                        strerror,
+                        space.unicode_w(space.repr(self.w_filename)),
+                        space.unicode_w(space.repr(self.w_filename2))))
                 return space.wrap(u"[Errno %s] %s: %s" % (
                     errno,
                     strerror,
@@ -590,6 +607,7 @@
     errno    = readwrite_attrproperty_w('w_errno',    W_OSError),
     strerror = readwrite_attrproperty_w('w_strerror', W_OSError),
     filename = readwrite_attrproperty_w('w_filename', W_OSError),
+    filename2= readwrite_attrproperty_w('w_filename2',W_OSError),
     characters_written = GetSetProperty(W_OSError.descr_get_written,
                                         W_OSError.descr_set_written),
     )
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -374,6 +374,23 @@
         e = SubOSErrorWithNew("some message", baz="baz")
         assert e.baz == "baz"
         assert e.args == ("some message",)
+        assert e.filename is None
+        assert e.filename2 is None
+
+    def test_oserror_3_args(self):
+        e = OSError(42, "bar", "baz")
+        assert e.args == (42, "bar")
+        assert e.filename == "baz"
+        assert e.filename2 is None
+        assert str(e) == "[Errno 42] bar: 'baz'"
+
+    def test_oserror_5_args(self):
+        # NB. argument 4 is only parsed on Windows
+        e = OSError(42, "bar", "baz", None, "bok")
+        assert e.args == (42, "bar")
+        assert e.filename == "baz"
+        assert e.filename2 == "bok"
+        assert str(e) == "[Errno 42] bar: 'baz' -> 'bok'"
 
     # Check the heuristic for print & exec covers significant cases
     # As well as placing some limits on false positives
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -484,7 +484,8 @@
     def __init__(self, space):
         self.stat_float_times = True
 
-def stat_float_times(space, w_value=None):
+ at unwrap_spec(newval=int)
+def stat_float_times(space, newval=-1):
     """stat_float_times([newval]) -> oldval
 
 Determine whether os.[lf]stat represents time stamps as float objects.
@@ -494,10 +495,10 @@
 """
     state = space.fromcache(StatState)
 
-    if w_value is None:
+    if newval == -1:
         return space.wrap(state.stat_float_times)
     else:
-        state.stat_float_times = space.bool_w(w_value)
+        state.stat_float_times = (newval != 0)
 
 
 @unwrap_spec(fd=c_int)
@@ -1008,7 +1009,7 @@
         else:
             dispatch_filename_2(rposix.rename)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
 
 @unwrap_spec(src_dir_fd=DirFD(rposix.HAVE_RENAMEAT),
         dst_dir_fd=DirFD(rposix.HAVE_RENAMEAT))
@@ -1115,7 +1116,7 @@
     rposix.kill(os.getpid(), signal.SIGABRT)
 
 @unwrap_spec(
-    src='fsencode', dst='fsencode',
+    src='fsencode', dst='fsencode',  # <- simpler: link() is never on Windows
     src_dir_fd=DirFD(rposix.HAVE_LINKAT), dst_dir_fd=DirFD(rposix.HAVE_LINKAT),
     follow_symlinks=bool)
 def link(
@@ -1144,7 +1145,7 @@
         else:
             rposix.link(src, dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror(space, e, filename=src, filename2=dst)
 
 
 @unwrap_spec(dir_fd=DirFD(rposix.HAVE_SYMLINKAT))
@@ -1171,7 +1172,7 @@
         else:
             dispatch_filename_2(rposix.symlink)(space, w_src, w_dst)
     except OSError as e:
-        raise wrap_oserror(space, e)
+        raise wrap_oserror2(space, e, w_filename=w_src, w_filename2=w_dst)
 
 
 @unwrap_spec(
@@ -2219,13 +2220,20 @@
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
-    return space.newbool(rposix.get_status_flags(fd) & rposix.O_NONBLOCK == 0)
+    try:
+        flags = rposix.get_status_flags(fd)
+    except OSError as e:
+        raise wrap_oserror(space, e)
+    return space.newbool(flags & rposix.O_NONBLOCK == 0)
 
 @unwrap_spec(fd=c_int, blocking=int)
 def set_blocking(space, fd, blocking):
-    flags = rposix.get_status_flags(fd)
-    if blocking:
-        flags &= ~rposix.O_NONBLOCK
-    else:
-        flags |= rposix.O_NONBLOCK
-    rposix.set_status_flags(fd, flags)
+    try:
+        flags = rposix.get_status_flags(fd)
+        if blocking:
+            flags &= ~rposix.O_NONBLOCK
+        else:
+            flags |= rposix.O_NONBLOCK
+        rposix.set_status_flags(fd, flags)
+    except OSError as e:
+        raise wrap_oserror(space, e)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -159,11 +159,14 @@
             st = posix.stat(path)
             assert isinstance(st.st_mtime, float)
             assert st[7] == int(st.st_atime)
+            assert posix.stat_float_times(-1) is True
 
             posix.stat_float_times(False)
             st = posix.stat(path)
             assert isinstance(st.st_mtime, int)
             assert st[7] == st.st_atime
+            assert posix.stat_float_times(-1) is False
+
         finally:
             posix.stat_float_times(current)
 
@@ -1148,6 +1151,11 @@
             assert posix.get_blocking(fd) is True
             posix.close(fd)
 
+        def test_blocking_error(self):
+            posix = self.posix
+            raises(OSError, posix.get_blocking, 1234567)
+            raises(OSError, posix.set_blocking, 1234567, True)
+
 
     def test_urandom(self):
         os = self.posix
@@ -1214,6 +1222,18 @@
         self.posix.RTLD_GLOBAL
         self.posix.RTLD_LOCAL
 
+    def test_error_message(self):
+        e = raises(OSError, self.posix.open, 'nonexistentfile1', 0)
+        assert str(e.value).endswith(": 'nonexistentfile1'")
+
+        e = raises(OSError, self.posix.link, 'nonexistentfile1', 'bok')
+        assert str(e.value).endswith(": 'nonexistentfile1' -> 'bok'")
+        e = raises(OSError, self.posix.rename, 'nonexistentfile1', 'bok')
+        assert str(e.value).endswith(": 'nonexistentfile1' -> 'bok'")
+
+        e = raises(OSError, self.posix.symlink, 'bok', '/nonexistentdir/boz')
+        assert str(e.value).endswith(": 'bok' -> '/nonexistentdir/boz'")
+
 
 class AppTestEnvironment(object):
     def setup_class(cls):
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -4,11 +4,11 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.unicodehelper import encode_utf8
 from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py
--- a/pypy/module/signal/__init__.py
+++ b/pypy/module/signal/__init__.py
@@ -3,6 +3,8 @@
 import os
 
 class Module(MixedModule):
+    applevel_name = '_signal'
+
     interpleveldefs = {
         'signal':              'interp_signal.signal',
         'getsignal':           'interp_signal.getsignal',
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -14,7 +14,7 @@
     def test_checksignals(self):
         space = self.space
         w_received = space.appexec([], """():
-            import signal
+            import _signal as signal
             received = []
             def myhandler(signum, frame):
                 received.append(signum)
@@ -40,7 +40,7 @@
     }
 
     def setup_class(cls):
-        cls.w_signal = cls.space.getbuiltinmodule('signal')
+        cls.w_signal = cls.space.getbuiltinmodule('_signal')
         cls.w_temppath = cls.space.wrap(
             str(py.test.ensuretemp("signal").join("foo.txt")))
         cls.w_appdirect = cls.space.wrap(cls.runappdirect)
@@ -92,7 +92,7 @@
         """
         Test that signal.signal returns SIG_DFL if that is the current handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
@@ -105,7 +105,7 @@
         """
         Test that signal.signal returns SIG_IGN if that is the current handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
@@ -119,7 +119,7 @@
         Test that signal.signal returns a Python object if one is the current
         handler.
         """
-        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import signal, SIGINT, SIG_DFL, SIG_IGN
         def installed(*a):
             pass
 
@@ -134,7 +134,7 @@
         """
         Test that signal.getsignal returns the currently installed handler.
         """
-        from signal import getsignal, signal, SIGINT, SIG_DFL, SIG_IGN
+        from _signal import getsignal, signal, SIGINT, SIG_DFL, SIG_IGN
 
         def handler(*a):
             pass
@@ -153,7 +153,7 @@
 
     def test_check_signum(self):
         import sys
-        from signal import getsignal, signal, NSIG
+        from _signal import getsignal, signal, NSIG
 
         # signum out of range fails
         raises(ValueError, getsignal, NSIG)
@@ -166,7 +166,7 @@
 
     def test_alarm(self):
         try:
-            from signal import alarm, signal, SIG_DFL, SIGALRM
+            from _signal import alarm, signal, SIG_DFL, SIGALRM
         except:
             skip('no alarm on this platform')
         import time
@@ -186,7 +186,7 @@
 
     def test_set_wakeup_fd(self):
         try:
-            import signal, posix, fcntl
+            import _signal as signal, posix, fcntl
         except ImportError:
             skip('cannot import posix or fcntl')
         def myhandler(signum, frame):
@@ -222,13 +222,13 @@
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
     def test_set_wakeup_fd_invalid(self):
-        import signal
+        import _signal as signal
         with open(self.temppath, 'wb') as f:
             fd = f.fileno()
         raises(ValueError, signal.set_wakeup_fd, fd)
 
     def test_siginterrupt(self):
-        import signal, os, time
+        import _signal as signal, os, time
         if not hasattr(signal, 'siginterrupt'):
             skip('non siginterrupt in signal')
         signum = signal.SIGUSR1
@@ -267,7 +267,7 @@
 
     def test_alarm_raise(self):
         try:
-            from signal import alarm, signal, SIG_DFL, SIGALRM
+            from _signal import alarm, signal, SIG_DFL, SIGALRM
         except ImportError:
             skip("no SIGALRM on this platform")
         import _socket
@@ -299,7 +299,7 @@
             py.test.skip("Unix only")
 
     def test_itimer_real(self):
-        import signal
+        import _signal as signal
 
         def sig_alrm(*args):
             self.called = True
@@ -316,7 +316,7 @@
         assert self.called
 
     def test_itimer_exc(self):
-        import signal
+        import _signal as signal
 
         raises(signal.ItimerError, signal.setitimer, -1, 0)
 
@@ -324,7 +324,7 @@
     spaceconfig = dict(usemodules=['signal', 'thread', 'time'])
 
     def test_pthread_kill(self):
-        import signal
+        import _signal as signal
         import _thread
         signum = signal.SIGUSR1
         def handler(signum, frame):
@@ -334,7 +334,7 @@
         raises(ZeroDivisionError, signal.pthread_kill, tid, signum)
         
     def test_sigwait(self):
-        import signal
+        import _signal as signal
         def handler(signum, frame):
             1/0
         signal.signal(signal.SIGALRM, handler)
@@ -343,7 +343,7 @@
         assert received == signal.SIGALRM
         
     def test_sigmask(self):
-        import signal, posix
+        import _signal as signal, posix
         signum1 = signal.SIGUSR1
         signum2 = signal.SIGUSR2
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -208,6 +208,7 @@
     def newunicode(self, x):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):
@@ -221,7 +222,6 @@
                 return w_some_obj()
             self._wrap_not_rpython(x)
         return w_some_obj()
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
@@ -314,10 +314,10 @@
         is_root(w_complex)
         return 1.1, 2.2
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     def decode_index(self, w_index_or_slice, seqlength):
         is_root(w_index_or_slice)
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -9,9 +9,9 @@
 
 def make_checker():
     check = []
+    @specialize.memo()
     def see():
         check.append(True)
-    see._annspecialcase_ = 'specialize:memo'
     return see, check
 
 def test_wrap_interp2app():
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -2,12 +2,12 @@
 import sys
 
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import INT_MAX
 from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
@@ -483,24 +483,28 @@
                                     "character code not in range(256)")
                     self.std_wp(s)
                 return
-            if space.isinstance_w(w_value, space.w_str):
-                s = space.str_w(w_value)
+            if not do_unicode:
+                if space.isinstance_w(w_value, space.w_str):
+                    s = space.str_w(w_value)
+                elif space.isinstance_w(w_value, space.w_bytearray):
+                    s = w_value.buffer_w(space, 0).as_str()
+                else:
+                    s = ''
                 if len(s) == 1:
                     self.std_wp(s)
                     return
-            elif space.isinstance_w(w_value, space.w_unicode):
-                if not do_unicode:
-                    raise NeedUnicodeFormattingError
-                ustr = space.unicode_w(w_value)
-                if len(ustr) == 1:
-                    self.std_wp(ustr)
-                    return
-            if do_unicode:
+                raise oefmt(space.w_TypeError, "%c requires int or single byte")
+            else:
+                if space.isinstance_w(w_value, space.w_unicode):
+                    ustr = space.unicode_w(w_value)
+                    if len(ustr) == 1:
+                        self.std_wp(ustr)
+                        return
                 raise oefmt(space.w_TypeError, "%c requires int or char")
-            else:
-                raise oefmt(space.w_TypeError, "%c requires int or single byte")
 
         def fmt_b(self, w_value):
+            if do_unicode:
+                self.unknown_fmtchar()
             space = self.space
             # cpython explicitly checks for bytes & bytearray
             if space.isinstance_w(w_value, space.w_bytes):
@@ -522,7 +526,7 @@
                 return
 
             raise oefmt(space.w_TypeError,
-                    "requires bytes, or an object that" \
+                    "requires bytes, or an object that "
                     "implements __bytes__, not '%T'", w_value)
 
     return StringFormatter
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -178,6 +178,7 @@
         raises(TypeError, '%c'.__mod__, ("bla",))
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))
+        raises(TypeError, '%c'.__mod__, b'A')
 
     def test___int__index__(self):
         class MyInt(object):
@@ -320,6 +321,10 @@
         f = 4
         raises(ValueError, '"%\u1234" % (f,)')
 
+    def test_invalid_b_with_unicode(self):
+        raises(ValueError, '"%b" % b"A"')
+        raises(ValueError, '"%b" % 42')
+
     def test_formatting_huge_precision(self):
         prec = 2**31
         format_string = u"%.{}f".format(prec)
@@ -368,6 +373,11 @@
         assert b"<%c>" % 48 == b"<0>"
         assert b"<%c>" % b"?" == b"<?>"
         raises(TypeError, 'b"<%c>" % "?"')
+        assert b"<%c>" % bytearray(b"?") == b"<?>"
+        class X:
+            def __bytes__(self):
+                return b'5'
+        raises(TypeError, 'b"<%c>" % X()')
 
     def test_bytes_bytes(self):
         assert b"<%b>" % b"123" == b"<123>"
@@ -421,6 +431,8 @@
         assert bytearray(b"<%c>") % 48 == bytearray(b"<0>")
         assert bytearray(b"<%c>") % b"?" == bytearray(b"<?>")
         raises(TypeError, 'bytearray(b"<%c>") % "?"')
+        assert bytearray(b"<%c>") % bytearray(b"?") == bytearray(b"<?>")
+        raises(TypeError, 'bytearray(b"<%c>") % memoryview(b"X")')
 
     def test_bytes_bytes(self):
         assert bytearray(b"<%b>") % b"123" == bytearray(b"<123>")
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -206,6 +206,10 @@
         pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
+
+    # XXX for Darwin running old versions of CPython 2.7.x
+    sysconfig.get_config_vars()
+
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler)  # XXX
     objects = []
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -2,6 +2,7 @@
 
 import types
 from collections import defaultdict
+from contextlib import contextmanager
 
 from rpython.tool.ansi_print import AnsiLogger
 from rpython.tool.pairtype import pair
@@ -83,22 +84,17 @@
         annmodel.TLS.check_str_without_nul = (
             self.translator.config.translation.check_str_without_nul)
 
-        flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
+        with self.using_policy(policy):
+            flowgraph, inputs_s = self.get_call_parameters(function, args_s)
 
         if main_entry_point:
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
-    def get_call_parameters(self, function, args_s, policy):
-        desc = self.bookkeeper.getdesc(function)
-        prevpolicy = self.policy
-        self.policy = policy
-        self.bookkeeper.enter(None)
-        try:
+    def get_call_parameters(self, function, args_s):
+        with self.bookkeeper.at_position(None):
+            desc = self.bookkeeper.getdesc(function)
             return desc.get_call_parameters(args_s)
-        finally:
-            self.bookkeeper.leave()
-            self.policy = prevpolicy
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
@@ -107,21 +103,29 @@
             # XXX hack
             annmodel.TLS.check_str_without_nul = (
                 self.translator.config.translation.check_str_without_nul)
-        graph, inputcells = self.get_call_parameters(function, args_s, policy)
-        self.build_graph_types(graph, inputcells, complete_now=False)
-        self.complete_helpers(policy)
+        with self.using_policy(policy):
+            graph, inputcells = self.get_call_parameters(function, args_s)
+            self.build_graph_types(graph, inputcells, complete_now=False)
+            self.complete_helpers()
         return graph
 
-    def complete_helpers(self, policy):
-        saved = self.policy, self.added_blocks
-        self.policy = policy
+    def complete_helpers(self):
+        saved = self.added_blocks
+        self.added_blocks = {}
         try:
-            self.added_blocks = {}
             self.complete()
             # invoke annotation simplifications for the new blocks
             self.simplify(block_subset=self.added_blocks)
         finally:
-            self.policy, self.added_blocks = saved
+            self.added_blocks = saved
+
+    @contextmanager
+    def using_policy(self, policy):
+        """A context manager that temporarily replaces the annotator policy"""
+        old_policy = self.policy
+        self.policy = policy
+        yield
+        self.policy = old_policy
 
     def build_graph_types(self, flowgraph, inputcells, complete_now=True):
         checkgraph(flowgraph)
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -9,6 +9,7 @@
 from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
+from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.model import (
     SomeOrderedDict, SomeString, SomeChar, SomeFloat, unionof, SomeInstance,
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
@@ -21,6 +22,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
+from rpython.annotator.specialize import memo
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
@@ -358,7 +360,7 @@
             return self.descs[obj_key]
         except KeyError:
             if isinstance(pyobj, types.FunctionType):
-                result = description.FunctionDesc(self, pyobj)
+                result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
                     raise Exception("ClassDesc for object not supported")
@@ -403,6 +405,23 @@
             self.descs[obj_key] = result
             return result
 
+    def newfuncdesc(self, pyfunc):
+        name = pyfunc.__name__
+        if hasattr(pyfunc, '_generator_next_method_of_'):
+            from rpython.flowspace.argument import Signature
+            signature = Signature(['entry'])     # haaaaaack
+            defaults = ()
+        else:
+            signature = cpython_code_signature(pyfunc.func_code)
+            defaults = pyfunc.func_defaults
+        # get the specializer based on the tag of the 'pyobj'
+        # (if any), according to the current policy
+        tag = getattr(pyfunc, '_annspecialcase_', None)
+        specializer = self.annotator.policy.get_specializer(tag)
+        if specializer is memo:
+            return description.MemoDesc(self, pyfunc, name, signature, defaults, specializer)
+        return description.FunctionDesc(self, pyfunc, name, signature, defaults, specializer)
+
     def getfrozen(self, pyobj):
         return description.FrozenDesc(self, pyobj)
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -608,7 +608,7 @@
             if mixin:
                 # make a new copy of the FunctionDesc for this class,
                 # but don't specialize further for all subclasses
-                funcdesc = FunctionDesc(self.bookkeeper, value)
+                funcdesc = self.bookkeeper.newfuncdesc(value)
                 self.classdict[name] = funcdesc
                 return
             # NB. if value is, say, AssertionError.__init__, then we
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -3,11 +3,10 @@
 from rpython.annotator.signature import (
     enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import FunctionGraph
-from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, s_ImpossibleValue
+from rpython.annotator.model import AnnotatorError, s_ImpossibleValue, unionof
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -117,7 +116,6 @@
         self.s_value = s_ImpossibleValue    # union of possible values
 
     def update(self, other):
-        from rpython.annotator.model import unionof
         self.descs.update(other.descs)
         self.read_locations.update(other.read_locations)
         self.s_value = unionof(self.s_value, other.s_value)
@@ -192,24 +190,12 @@
 class FunctionDesc(Desc):
     knowntype = types.FunctionType
 
-    def __init__(self, bookkeeper, pyobj=None,
-                 name=None, signature=None, defaults=None,
+    def __init__(self, bookkeeper, pyobj, name, signature, defaults,
                  specializer=None):
         super(FunctionDesc, self).__init__(bookkeeper, pyobj)
-        if name is None:
-            name = pyobj.func_name
-        if signature is None:
-            if hasattr(pyobj, '_generator_next_method_of_'):
-                from rpython.flowspace.argument import Signature
-                signature = Signature(['entry'])     # haaaaaack
-                defaults = ()
-            else:
-                signature = cpython_code_signature(pyobj.func_code)
-        if defaults is None:
-            defaults = pyobj.func_defaults
         self.name = name
         self.signature = signature
-        self.defaults = defaults or ()
+        self.defaults = defaults if defaults is not None else ()
         # 'specializer' is a function with the following signature:
         #      specializer(funcdesc, args_s) => graph
         #                                 or => s_result (overridden/memo cases)
@@ -288,12 +274,43 @@
                 getattr(self.bookkeeper, "position_key", None) is not None):
             _, block, i = self.bookkeeper.position_key
             op = block.operations[i]
-        if self.specializer is None:
-            # get the specializer based on the tag of the 'pyobj'
-            # (if any), according to the current policy
-            tag = getattr(self.pyobj, '_annspecialcase_', None)
-            policy = self.bookkeeper.annotator.policy
-            self.specializer = policy.get_specializer(tag)
+        self.normalize_args(inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
+
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
+        # Some specializations may break the invariant of returning
+        # annotations that are always more general than the previous time.
+        # We restore it here:
+        result = unionof(result, s_previous_result)
+        return result
+
+    def normalize_args(self, inputs_s):
+        """
+        Canonicalize argument annotations into the exact parameter
+        annotations of a specific specialized graph.
+
+        Note: this method has no return value but mutates its argument instead.
+        """
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
@@ -304,39 +321,9 @@
                 from rpython.annotator.signature import Sig
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
-            enforceargs(self, inputcells)  # can modify inputcells in-place
+            enforceargs(self, inputs_s)  # can modify inputs_s in-place
         if signature:
-            enforce_signature_args(self, signature[0], inputcells)  # mutates inputcells
-        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
-            return self.specializer(self, inputcells, op)
-        else:
-            return self.specializer(self, inputcells)
-
-    def pycall(self, whence, args, s_previous_result, op=None):
-        inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
-        # Some specializations may break the invariant of returning
-        # annotations that are always more general than the previous time.
-        # We restore it here:
-        from rpython.annotator.model import unionof
-        result = unionof(result, s_previous_result)
-        return result
+            enforce_signature_args(self, signature[0], inputs_s)  # mutates inputs_s
 
     def get_graph(self, args, op):
         inputs_s = self.parse_arguments(args)
@@ -405,6 +392,16 @@
 
             return s_sigs
 
+class MemoDesc(FunctionDesc):
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        s_result = self.specialize(inputcells, op)
+        if isinstance(s_result, FunctionGraph):
+            s_result = s_result.getreturnvar().annotation
+        s_result = unionof(s_result, s_previous_result)
+        return s_result
+
+
 class MethodDesc(Desc):
     knowntype = types.MethodType
 
diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py
--- a/rpython/annotator/specialize.py
+++ b/rpython/annotator/specialize.py
@@ -3,11 +3,13 @@
 
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import Block, Link, Variable, SpaceOperation
+from rpython.flowspace.model import Block, Link, Variable
 from rpython.flowspace.model import checkgraph
 from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.flowspace.argument import Signature
+from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
+from rpython.annotator.model import unionof
 
 def flatten_star_args(funcdesc, args_s):
     argnames, vararg, kwarg = funcdesc.signature
@@ -127,7 +129,6 @@
     def finish(self):
         if self.do_not_process:
             return
-        from rpython.annotator.model import unionof
         assert self.graph is None, "MemoTable already finished"
         # list of which argument positions can take more than one value
         example_args, example_value = self.table.iteritems().next()
@@ -246,34 +247,36 @@
             args_s.append(unionof(*values_s))
         annotator.addpendinggraph(self.graph, args_s)
 
+def all_values(s):
+    """Return the exhaustive list of possible values matching annotation `s`.
 
-def memo(funcdesc, arglist_s):
-    from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
-    from rpython.annotator.model import unionof
+    Raises `AnnotatorError` if no such (reasonably small) finite list exists.
+    """
+    if s.is_constant():
+        return [s.const]
+    elif isinstance(s, SomePBC):
+        values = []
+        assert not s.can_be_None, "memo call: cannot mix None and PBCs"
+        for desc in s.descriptions:
+            if desc.pyobj is None:
+                raise annmodel.AnnotatorError(
+                    "memo call with a class or PBC that has no "
+                    "corresponding Python object (%r)" % (desc,))
+            values.append(desc.pyobj)
+        return values
+    elif isinstance(s, SomeImpossibleValue):
+        return []
+    elif isinstance(s, SomeBool):
+        return [False, True]
+    else:
+        raise annmodel.AnnotatorError("memo call: argument must be a class "
+                                        "or a frozen PBC, got %r" % (s,))
+
+def memo(funcdesc, args_s):
     # call the function now, and collect possible results
-    argvalues = []
-    for s in arglist_s:
-        if s.is_constant():
-            values = [s.const]
-        elif isinstance(s, SomePBC):
-            values = []
-            assert not s.can_be_None, "memo call: cannot mix None and PBCs"
-            for desc in s.descriptions:
-                if desc.pyobj is None:
-                    raise annmodel.AnnotatorError(
-                        "memo call with a class or PBC that has no "
-                        "corresponding Python object (%r)" % (desc,))
-                values.append(desc.pyobj)
-        elif isinstance(s, SomeImpossibleValue):
-            return s    # we will probably get more possible args later
-        elif isinstance(s, SomeBool):
-            values = [False, True]
-        else:
-            raise annmodel.AnnotatorError("memo call: argument must be a class "
-                                          "or a frozen PBC, got %r" % (s,))
-        argvalues.append(values)
+
     # the list of all possible tuples of arguments to give to the memo function
-    possiblevalues = cartesian_product(argvalues)
+    possiblevalues = cartesian_product([all_values(s_arg) for s_arg in args_s])
 
     # a MemoTable factory -- one MemoTable per family of arguments that can
     # be called together, merged via a UnionFind.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2936,6 +2936,12 @@
         self._pyobj(pyobject).ob_pypy_link = objint
         # there is no rrc_o_dict
 
+    def rawrefcount_mark_deallocating(self, gcobj, pyobject):
+        ll_assert(self.rrc_enabled, "rawrefcount.init not called")
+        obj = llmemory.cast_ptr_to_adr(gcobj)   # should be a prebuilt obj
+        objint = llmemory.cast_adr_to_int(obj, "symbolic")
+        self._pyobj(pyobject).ob_pypy_link = objint
+
     def rawrefcount_from_obj(self, gcobj):
         obj = llmemory.cast_ptr_to_adr(gcobj)
         if self.is_in_nursery(obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -476,6 +476,10 @@
                 GCClass.rawrefcount_create_link_pyobj,
                 [s_gc, s_gcref, SomeAddress()],
                 annmodel.s_None)
+            self.rawrefcount_mark_deallocating = getfn(
+                GCClass.rawrefcount_mark_deallocating,
+                [s_gc, s_gcref, SomeAddress()],
+                annmodel.s_None)
             self.rawrefcount_from_obj_ptr = getfn(
                 GCClass.rawrefcount_from_obj, [s_gc, s_gcref], SomeAddress(),
                 inline = True)
@@ -1281,6 +1285,14 @@
                   [self.rawrefcount_create_link_pyobj_ptr, self.c_const_gc,
                    v_gcobj, v_pyobject])
 
+    def gct_gc_rawrefcount_mark_deallocating(self, hop):
+        [v_gcobj, v_pyobject] = hop.spaceop.args
+        assert v_gcobj.concretetype == llmemory.GCREF
+        assert v_pyobject.concretetype == llmemory.Address
+        hop.genop("direct_call",
+                  [self.rawrefcount_mark_deallocating, self.c_const_gc,
+                   v_gcobj, v_pyobject])
+
     def gct_gc_rawrefcount_from_obj(self, hop):
         [v_gcobj] = hop.spaceop.args
         assert v_gcobj.concretetype == llmemory.GCREF
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -3,7 +3,7 @@
 import py
 
 from rpython.rlib.nonconst import NonConstant
-from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython
+from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize, not_rpython, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.sourcetools import rpython_wrapper
@@ -1221,7 +1221,8 @@
         x = jit.conditional_call_elidable(self.cache, _compute_and_cache, ...)
 
     """
-    if we_are_jitted():
+    if we_are_translated() and we_are_jitted():
+        #^^^ the occasional test patches we_are_jitted() to True
         return _jit_conditional_call_value(value, function, *args)
     else:
         if isinstance(value, int):
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -25,6 +25,10 @@
          lltype.Unsigned, widen it to lltype.Signed.
          Useful because the translator doesn't support
          arithmetic on the smaller types.
+ovfcheck_int32_add/sub/mul(x, y)
+         perform an add/sub/mul between two regular integers,
+         but assumes that they fit inside signed 32-bit ints
+         and raises OverflowError if the result no longer does
 
 These are meant to be erased by translation, r_uint
 in the process should mark unsigned values, ovfcheck should
@@ -796,6 +800,47 @@
         return longlong2float(rffi.cast(rffi.LONGLONG, res))
     return rffi.cast(T, res)
 
+if sys.maxint == 2147483647:
+    def ovfcheck_int32_add(x, y):
+        return ovfcheck(x + y)
+    def ovfcheck_int32_sub(x, y):
+        return ovfcheck(x - y)
+    def ovfcheck_int32_mul(x, y):
+        return ovfcheck(x * y)
+else:
+    def ovfcheck_int32_add(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x + y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_sub(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x - y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
+    def ovfcheck_int32_mul(x, y):
+        """x and y are assumed to fit inside the 32-bit rffi.INT;
+        raises OverflowError if the result doesn't fit rffi.INT"""
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        x = rffi.cast(lltype.Signed, x)
+        y = rffi.cast(lltype.Signed, y)
+        z = x * y
+        if z != rffi.cast(lltype.Signed, rffi.cast(rffi.INT, z)):
+            raise OverflowError
+        return z
+
 
 # String parsing support
 # ---------------------------
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -36,6 +36,7 @@
     _pypy2ob = {}
     _pypy2ob_rev = {}
     _d_list = []
+    _d_marker = None
     _dealloc_trigger_callback = dealloc_trigger_callback
 
 @not_rpython
@@ -62,6 +63,14 @@
     _o_list.append(ob)
 
 @not_rpython
+def mark_deallocating(marker, ob):
+    """mark the PyObject as deallocating, by storing 'marker'
+    inside its ob_pypy_link field"""
+    assert ob._obj not in _pypy2ob_rev
+    assert not ob.c_ob_pypy_link
+    ob.c_ob_pypy_link = _build_pypy_link(marker)
+
+ at not_rpython
 def from_obj(OB_PTR_TYPE, p):
     ob = _pypy2ob.get(p)
     if ob is None:
@@ -221,7 +230,7 @@
 
 
 class Entry(ExtRegistryEntry):
-    _about_ = (create_link_pypy, create_link_pyobj)
+    _about_ = (create_link_pypy, create_link_pyobj, mark_deallocating)
 
     def compute_result_annotation(self, s_p, s_ob):
         pass
@@ -231,6 +240,8 @@
             name = 'gc_rawrefcount_create_link_pypy'
         elif self.instance is create_link_pyobj:
             name = 'gc_rawrefcount_create_link_pyobj'
+        elif self.instance is mark_deallocating:
+            name = 'gc_rawrefcount_mark_deallocating'
         v_p, v_ob = hop.inputargs(*hop.args_r)
         hop.exception_cannot_occur()
         hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)])
diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py
--- a/rpython/rlib/rjitlog/rjitlog.py
+++ b/rpython/rlib/rjitlog/rjitlog.py
@@ -13,6 +13,7 @@
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.history import ConstInt, ConstFloat
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import r_longlong
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rlib.objectmodel import compute_unique_id, always_inline
 from rpython.rlib.objectmodel import we_are_translated, specialize
@@ -113,6 +114,7 @@
 
 @always_inline
 def encode_le_64bit(val):
+    val = r_longlong(val)     # force 64-bit, even on 32-bit
     return ''.join([chr((val >> 0) & 0xff),
                     chr((val >> 8) & 0xff),
                     chr((val >> 16) & 0xff),
diff --git a/rpython/rlib/rurandom.py b/rpython/rlib/rurandom.py
--- a/rpython/rlib/rurandom.py
+++ b/rpython/rlib/rurandom.py
@@ -75,16 +75,6 @@
 
             return rffi.charpsize2str(rffi.cast(rffi.CCHARP, buf), n)
 
-elif 0:  # __VMS
-    from rpython.rlib.ropenssl import libssl_RAND_pseudo_bytes
-    def init_urandom():
-        pass
-
-    def urandom(context, n):
-        with rffi.scoped_alloc_buffer(n) as buf:
-            if libssl_RAND_pseudo_bytes(self.raw, n) < 0:
-                raise ValueError("RAND_pseudo_bytes")
-            return buf.str(n)
 else:  # Posix implementation
     @not_rpython
     def init_urandom():
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -126,13 +126,14 @@
 // how we'd get the PC (using StackWalk64?)
 //    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
 
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
+// #include "base/logging.h"   // for RAW_LOG
+// #ifndef HAVE_CYGWIN_SIGNAL_H
+// typedef int ucontext_t;
+// #endif
 
 intptr_t GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  // RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  fprintf(stderr, "GetPC is not yet implemented on Windows\n");
   return NULL;
 }
 
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -675,3 +675,11 @@
     assert a ^ r_uint32(42) == "a^42"
     assert r_uint32(42) ** a == "42**a"
     assert a ** r_uint32(42) == "a**42"
+
+def test_ovfcheck_int32():
+    assert ovfcheck_int32_add(-2**30, -2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_add, 2**30, 2**30)
+    assert ovfcheck_int32_sub(-2**30, 2**30) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_sub, 2**30, -2**30)
+    assert ovfcheck_int32_mul(-2**16, 2**15) == -2**31
+    py.test.raises(OverflowError, ovfcheck_int32_mul, -2**16, -2**15)
diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py
--- a/rpython/rlib/test/test_rawrefcount.py
+++ b/rpython/rlib/test/test_rawrefcount.py
@@ -212,6 +212,15 @@
         assert rawrefcount.to_obj(W_Root, ob) == p
         lltype.free(ob, flavor='raw')
 
+    def test_mark_deallocating(self):
+        ob = lltype.malloc(PyObjectS, flavor='raw', zero=True)
+        w_marker = W_Root(42)
+        rawrefcount.mark_deallocating(w_marker, ob)
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        rawrefcount._collect()
+        assert rawrefcount.to_obj(W_Root, ob) is w_marker
+        lltype.free(ob, flavor='raw')
+
 
 class TestTranslated(StandaloneTests):
 
@@ -222,6 +231,7 @@
         state.seen = []
         def dealloc_trigger():
             state.seen.append(1)
+        w_marker = W_Root(-1)
 
         def make_p():
             p = W_Root(42)
@@ -257,6 +267,13 @@
             if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS):
                 print "NEXT_DEAD second time != NULL"
                 return 1
+            if rawrefcount.to_obj(W_Root, ob) is not None:
+                print "to_obj(dead) is not None?"
+                return 1
+            rawrefcount.mark_deallocating(w_marker, ob)
+            if rawrefcount.to_obj(W_Root, ob) is not w_marker:
+                print "to_obj(marked-dead) is not w_marker"
+                return 1
             print "OK!"
             lltype.free(ob, flavor='raw')
             return 0
diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py
--- a/rpython/rtyper/annlowlevel.py
+++ b/rpython/rtyper/annlowlevel.py
@@ -138,11 +138,12 @@
         # get the graph of the mix-level helper ll_function and prepare it for
         # being annotated.  Annotation and RTyping should be done in a single shot
         # at the end with finish().
-        graph, args_s = self.rtyper.annotator.get_call_parameters(
-            ll_function, args_s, policy = self.policy)
+        ann = self.rtyper.annotator
+        with ann.using_policy(self.policy):
+            graph, args_s = ann.get_call_parameters(ll_function, args_s)
         for v_arg, s_arg in zip(graph.getargs(), args_s):
-            self.rtyper.annotator.setbinding(v_arg, s_arg)
-        self.rtyper.annotator.setbinding(graph.getreturnvar(), s_result)
+            ann.setbinding(v_arg, s_arg)
+        ann.setbinding(graph.getreturnvar(), s_result)
         #self.rtyper.annotator.annotated[graph.returnblock] = graph
         self.pending.append((ll_function, graph, args_s, s_result))
         return graph
@@ -224,16 +225,17 @@
         bk = ann.bookkeeper
         translator = ann.translator
         original_graph_count = len(translator.graphs)
-        for ll_function, graph, args_s, s_result in self.pending:
-            # mark the return block as already annotated, because the return var
-            # annotation was forced in getgraph() above.  This prevents temporary
-            # less general values reaching the return block from crashing the
-            # annotator (on the assert-that-new-binding-is-not-less-general).
-            ann.annotated[graph.returnblock] = graph
-            s_function = bk.immutablevalue(ll_function)
-            bk.emulate_pbc_call(graph, s_function, args_s)
-            self.newgraphs.add(graph)
-        ann.complete_helpers(self.policy)
+        with ann.using_policy(self.policy):
+            for ll_function, graph, args_s, s_result in self.pending:
+                # mark the return block as already annotated, because the return var
+                # annotation was forced in getgraph() above.  This prevents temporary
+                # less general values reaching the return block from crashing the
+                # annotator (on the assert-that-new-binding-is-not-less-general).
+                ann.annotated[graph.returnblock] = graph
+                s_function = bk.immutablevalue(ll_function)
+                bk.emulate_pbc_call(graph, s_function, args_s)
+                self.newgraphs.add(graph)
+            ann.complete_helpers()
         for ll_function, graph, args_s, s_result in self.pending:
             s_real_result = ann.binding(graph.getreturnvar())
             if s_real_result != s_result:
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -959,6 +959,9 @@
     def op_gc_rawrefcount_create_link_pypy(self, *args):
         raise NotImplementedError("gc_rawrefcount_create_link_pypy")
 
+    def op_gc_rawrefcount_mark_deallocating(self, *args):
+        raise NotImplementedError("gc_rawrefcount_mark_deallocating")
+
     def op_do_malloc_fixedsize(self):
         raise NotImplementedError("do_malloc_fixedsize")
     def op_do_malloc_fixedsize_clear(self):
diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py
--- a/rpython/rtyper/lltypesystem/llmemory.py
+++ b/rpython/rtyper/lltypesystem/llmemory.py
@@ -384,7 +384,6 @@
 def _sizeof_none(TYPE):
     assert not TYPE._is_varsize()
     return ItemOffset(TYPE)
-_sizeof_none._annspecialcase_ = 'specialize:memo'
 
 @specialize.memo()
 def _internal_array_field(TYPE):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -490,6 +490,7 @@
     'gc_rawrefcount_init':              LLOp(),
     'gc_rawrefcount_create_link_pypy':  LLOp(),
     'gc_rawrefcount_create_link_pyobj': LLOp(),
+    'gc_rawrefcount_mark_deallocating': LLOp(),
     'gc_rawrefcount_from_obj':          LLOp(sideeffects=False),
     'gc_rawrefcount_to_obj':            LLOp(sideeffects=False),
 
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -7,7 +7,7 @@
 from rpython.rtyper.lltypesystem.llmemory import itemoffsetof
 from rpython.rtyper.llannotation import lltype_to_annotation
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, specialize
 from rpython.rlib.objectmodel import keepalive_until_here, enforceargs
 from rpython.rlib import rarithmetic, rgc
 from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -74,7 +74,7 @@
 def llexternal(name, args, result, _callable=None,
                compilation_info=ExternalCompilationInfo(),
                sandboxsafe=False, releasegil='auto',
-               _nowrapper=False, calling_conv='c',
+               _nowrapper=False, calling_conv=None,
                elidable_function=False, macro=None,
                random_effects_on_gcobjs='auto',
                save_err=RFFI_ERR_NONE):
@@ -96,7 +96,17 @@
                 we consider that the function is really short-running and
                 don't bother releasing the GIL.  An explicit True or False
                 overrides this logic.
+
+    calling_conv: if 'unknown' or 'win', the C function is not directly seen
+                  by the JIT.  If 'c', it can be seen (depending on
+                  releasegil=False).  For tests only, or if _nowrapper,
+                  it defaults to 'c'.
     """
+    if calling_conv is None:
+        if sys.platform == 'win32' and not _nowrapper:
+            calling_conv = 'unknown'
+        else:
+            calling_conv = 'c'
     if _callable is not None:
         assert callable(_callable)
     ext_type = lltype.FuncType(args, result)
@@ -107,7 +117,8 @@
             _callable = generate_macro_wrapper(
                 name, macro, ext_type, compilation_info)
         else:
-            _callable = ll2ctypes.LL2CtypesCallable(ext_type, calling_conv)
+            _callable = ll2ctypes.LL2CtypesCallable(ext_type,
+                'c' if calling_conv == 'unknown' else calling_conv)
     else:
         assert macro is None, "'macro' is useless if you specify '_callable'"
     if elidable_function:
@@ -205,11 +216,17 @@
     else:
         # if we don't have to invoke the GIL handling, we can just call
         # the low-level function pointer carelessly
-        if macro is None and save_err == RFFI_ERR_NONE:
+        # ...well, unless it's a macro, in which case we still have
+        # to hide it from the JIT...
+        need_wrapper = (macro is not None or save_err != RFFI_ERR_NONE)
+        # ...and unless we're on Windows and the calling convention is
+        # 'win' or 'unknown'
+        if calling_conv != 'c':
+            need_wrapper = True
+        #
+        if not need_wrapper:
             call_external_function = funcptr
         else:
-            # ...well, unless it's a macro, in which case we still have
-            # to hide it from the JIT...
             argnames = ', '.join(['a%d' % i for i in range(len(args))])
             source = py.code.Source("""
                 def call_external_function(%(argnames)s):
@@ -308,10 +325,6 @@
     # for debugging, stick ll func ptr to that
     wrapper._ptr = funcptr
     wrapper = func_with_new_name(wrapper, name)
-
-    if calling_conv != "c":
-        wrapper = jit.dont_look_inside(wrapper)
-
     return wrapper
 
 
@@ -1288,10 +1301,12 @@
 c_memcpy = llexternal("memcpy",
             [VOIDP, VOIDP, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            calling_conv='c',
         )
 c_memset = llexternal("memset",
             [VOIDP, lltype.Signed, SIZE_T],
             lltype.Void,
-            releasegil=False
+            releasegil=False,
+            calling_conv='c',
         )
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -362,9 +362,9 @@
     def get_concrete_llfn(self, s_pbc, args_s, op):
         bk = self.rtyper.annotator.bookkeeper
         funcdesc, = s_pbc.descriptions
-        args = simple_args(args_s)
         with bk.at_position(None):
-            graph = funcdesc.get_graph(args, op)
+            argspec = simple_args(args_s)
+            graph = funcdesc.get_graph(argspec, op)

From pypy.commits at gmail.com  Tue Nov 29 09:36:31 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 29 Nov 2016 06:36:31 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: blind fix,
 cpython test suite tests the repr. of sockets, it renamed protocol to proto
Message-ID: <583d926f.4438c20a.48ef1.f72d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r88739:278c99b028ec
Date: 2016-11-29 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/278c99b028ec/

Log:	blind fix, cpython test suite tests the repr. of sockets, it renamed
	protocol to proto

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -216,7 +216,7 @@
     def descr_repr(self, space):
         fd = intmask(self.sock.fd)  # Force to signed type even on Windows.
         return space.wrap("<socket object, fd=%d, family=%d,"
-                          " type=%d, protocol=%d>" %
+                          " type=%d, proto=%d>" %
                           (fd, self.sock.family,
                            self.sock.type, self.sock.proto))
 

From pypy.commits at gmail.com  Tue Nov 29 10:18:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:18:05 -0800 (PST)
Subject: [pypy-commit] pypy default: Use a dont_look_inside for now for the
 JIT
Message-ID: <583d9c2d.c4811c0a.a8612.186e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88740:1312ed9f77fe
Date: 2016-11-29 15:11 +0100
http://bitbucket.org/pypy/pypy/changeset/1312ed9f77fe/

Log:	Use a dont_look_inside for now for the JIT

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -527,6 +527,7 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
+ at jit.dont_look_inside
 def may_ignore_finalizer(obj):
     """Optimization hint: says that it is valid for any finalizer
     for 'obj' to be ignored, depending on the GC."""

From pypy.commits at gmail.com  Tue Nov 29 10:18:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:18:07 -0800 (PST)
Subject: [pypy-commit] pypy default: Use rgc.may_ignore_finalizer() inside
 PyPy where it makes sense to
Message-ID: <583d9c2f.0a22c20a.3594d.129f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88741:b31a6c797a0e
Date: 2016-11-29 15:27 +0100
http://bitbucket.org/pypy/pypy/changeset/b31a6c797a0e/

Log:	Use rgc.may_ignore_finalizer() inside PyPy where it makes sense to

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -183,6 +183,14 @@
                 assert self._finalize_.im_func is not W_Root._finalize_.im_func
             space.finalizer_queue.register_finalizer(self)
 
+    def may_unregister_rpython_finalizer(self, space):
+        """Optimization hint only: if there is no user-defined __del__()
+        method, pass the hint ``don't call any finalizer'' to rgc.
+        """
+        if not self.getclass(space).hasuserdel:
+            from rpython.rlib import rgc
+            rgc.may_ignore_finalizer(self)
+
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.pyopcode import LoopBlock
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 
 
 class GeneratorIterator(W_Root):
@@ -103,11 +103,11 @@
                 w_result = frame.execute_frame(w_arg, operr)
             except OperationError:
                 # errors finish a frame
-                self.frame = None
+                self.frame_is_finished()
                 raise
             # if the frame is now marked as finished, it was RETURNed from
             if frame.frame_finished_execution:
-                self.frame = None
+                self.frame_is_finished()
                 raise OperationError(space.w_StopIteration, space.w_None)
             else:
                 return w_result     # YIELDed
@@ -209,7 +209,7 @@
             finally:
                 frame.f_backref = jit.vref_None
                 self.running = False
-                self.frame = None
+                self.frame_is_finished()
         return unpack_into
     unpack_into = _create_unpack_into()
     unpack_into_w = _create_unpack_into()
@@ -228,6 +228,10 @@
                     break
                 block = block.previous
 
+    def frame_is_finished(self):
+        self.frame = None
+        rgc.may_ignore_finalizer(self)
+
 
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -397,7 +397,7 @@
         space = self.space
         if space.is_none(w_destructor):
             if isinstance(self, W_CDataGCP):
-                self.w_destructor = None
+                self.detach_destructor()
                 return space.w_None
             raise oefmt(space.w_TypeError,
                         "Can remove destructor only on a object "
@@ -604,6 +604,10 @@
             self.w_destructor = None
             self.space.call_function(w_destructor, self.w_original_cdata)
 
+    def detach_destructor(self):
+        self.w_destructor = None
+        self.may_unregister_rpython_finalizer(self.space)
+
 
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -55,6 +55,7 @@
         if not libhandle:
             raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed",
                         self.libname)
+        self.may_unregister_rpython_finalizer(self.ffi.space)
 
         # Clear the dict to force further accesses to do cdlopen_fetch()
         # again, and fail because the library was closed.  Note that the
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -172,6 +172,7 @@
             self.newlines = self.stream.getnewlines()
             self.stream = None
             self.fd = -1
+            self.may_unregister_rpython_finalizer(self.space)
             openstreams = getopenstreams(self.space)
             try:
                 del openstreams[stream]
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -233,6 +233,7 @@
         except SocketError:
             # cpython doesn't return any errors on close
             pass
+        self.may_unregister_rpython_finalizer(space)
 
     def connect_w(self, space, w_addr):
         """connect(address)
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -79,6 +79,7 @@
 
 class W_Epoll(W_Root):
     def __init__(self, space, epfd):
+        self.space = space
         self.epfd = epfd
         self.register_finalizer(space)
 
@@ -113,6 +114,7 @@
         if not self.get_closed():
             socketclose(self.epfd)
             self.epfd = -1
+            self.may_unregister_rpython_finalizer(self.space)
 
     def epoll_ctl(self, space, ctl, w_fd, eventmask, ignore_ebadf=False):
         fd = space.c_filedescriptor_w(w_fd)
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -108,6 +108,7 @@
 
 class W_Kqueue(W_Root):
     def __init__(self, space, kqfd):
+        self.space = space
         self.kqfd = kqfd
         self.register_finalizer(space)
 
@@ -132,6 +133,7 @@
             kqfd = self.kqfd
             self.kqfd = -1
             socketclose_no_errno(kqfd)
+            self.may_unregister_rpython_finalizer(self.space)
 
     def check_closed(self, space):
         if self.get_closed():
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -202,6 +202,7 @@
                 if mode == rzlib.Z_FINISH:    # release the data structures now
                     rzlib.deflateEnd(self.stream)
                     self.stream = rzlib.null_stream
+                    self.may_unregister_rpython_finalizer(space)
             finally:
                 self.unlock()
         except rzlib.RZlibError as e:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -66,15 +66,16 @@
 class W_MyType(W_MyObject):
     name = "foobar"
     flag_map_or_seq = '?'
+    hasuserdel = False
 
     def __init__(self):
         self.mro_w = [w_some_obj(), w_some_obj()]
         self.dict_w = {'__str__': w_some_obj()}
+        self.hasuserdel = True
 
     def get_module(self):
         return w_some_obj()
 
-
     def getname(self, space):
         return self.name
 
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -4,7 +4,7 @@
 """
 
 import os, stat, errno, sys
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rgc
 from rpython.rlib.objectmodel import enforceargs
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder
@@ -294,6 +294,7 @@
         if ll_file:
             # double close is allowed
             self._ll_file = lltype.nullptr(FILEP.TO)
+            rgc.may_ignore_finalizer(self)
             do_close = self._close2[0]
             try:
                 if do_close:

From pypy.commits at gmail.com  Tue Nov 29 10:18:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:18:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix for -A tests
Message-ID: <583d9c31.90a81c0a.b6e00.1e64@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88742:5ddd2ea65793
Date: 2016-11-29 16:06 +0100
http://bitbucket.org/pypy/pypy/changeset/5ddd2ea65793/

Log:	fix for -A tests

diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -40,23 +40,22 @@
     }
 
     def setup_class(cls):
-        cls.w_signal = cls.space.getbuiltinmodule('_signal')
         cls.w_temppath = cls.space.wrap(
             str(py.test.ensuretemp("signal").join("foo.txt")))
         cls.w_appdirect = cls.space.wrap(cls.runappdirect)
 
     def test_exported_names(self):
-        import os
-        self.signal.__dict__   # crashes if the interpleveldefs are invalid
+        import os, _signal
+        _signal.__dict__   # crashes if the interpleveldefs are invalid
         if os.name == 'nt':
-            assert self.signal.CTRL_BREAK_EVENT == 1
-            assert self.signal.CTRL_C_EVENT == 0
+            assert _signal.CTRL_BREAK_EVENT == 1
+            assert _signal.CTRL_C_EVENT == 0
 
     def test_basics(self):
-        import types, os
+        import types, os, _signal
         if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
             skip("requires os.kill() and os.getpid()")
-        signal = self.signal   # the signal module to test
+        signal = _signal   # the signal module to test
         if not hasattr(signal, 'SIGUSR1'):
             skip("requires SIGUSR1 in signal")
         signum = signal.SIGUSR1

From pypy.commits at gmail.com  Tue Nov 29 10:18:12 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:18:12 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583d9c34.8ab81c0a.bb224.16e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88744:5b048af062a4
Date: 2016-11-29 16:17 +0100
http://bitbucket.org/pypy/pypy/changeset/5b048af062a4/

Log:	merge heads

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -216,7 +216,7 @@
     def descr_repr(self, space):
         fd = intmask(self.sock.fd)  # Force to signed type even on Windows.
         return space.wrap("<socket object, fd=%d, family=%d,"
-                          " type=%d, protocol=%d>" %
+                          " type=%d, proto=%d>" %
                           (fd, self.sock.family,
                            self.sock.type, self.sock.proto))
 

From pypy.commits at gmail.com  Tue Nov 29 10:18:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:18:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Make generic the code to find the
 app-level name of a builtin module
Message-ID: <583d9c32.c89cc20a.e6962.f545@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88743:421d7a728d21
Date: 2016-11-29 16:16 +0100
http://bitbucket.org/pypy/pypy/changeset/421d7a728d21/

Log:	Make generic the code to find the app-level name of a builtin module
	from its rpython-level name

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -21,13 +21,6 @@
 
 pypyroot = os.path.dirname(pypydir)
 
-RENAMED_USEMODULES = dict(
-    _winreg='winreg',
-    exceptions='builtins',
-    struct='_struct',
-    thread='_thread',
-    operator='_operator',
-    )
 
 class AppError(Exception):
     def __init__(self, excinfo):
@@ -63,6 +56,11 @@
         return repr(value)
 
 
+def _rename_module(name):
+    mod = __import__("pypy.module." + name, globals(), locals(), ['Module'])
+    return mod.Module.applevel_name or name
+
+
 def run_with_python(python_, target_, usemodules, **definitions):
     if python_ is None:
         py.test.skip("Cannot find the default python3 interpreter to run with -A")
@@ -133,8 +131,7 @@
 
     check_usemodules = ''
     if usemodules:
-        usemodules = [str(RENAMED_USEMODULES.get(name, name))
-                      for name in usemodules]
+        usemodules = [_rename_module(name) for name in usemodules]
         check_usemodules = """\
     missing = set(%r).difference(sys.builtin_module_names)
     if missing:
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -135,3 +135,12 @@
     info = raises(ZeroDivisionError, "x/0")
     assert info.type is ZeroDivisionError
     assert isinstance(info.value, ZeroDivisionError)
+
+def test_rename_module():
+    from pypy.tool.pytest.apptest import _rename_module
+    assert _rename_module("sys") == "sys"
+    if sys.platform == "win32":
+        assert _rename_module("_winreg") == "winreg"
+    assert _rename_module("struct") == "_struct"
+    assert _rename_module("operator") == "_operator"
+    assert _rename_module("signal") == "_signal"

From pypy.commits at gmail.com  Tue Nov 29 10:35:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:35:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <583da031.45f6c20a.f25f.fb0c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88745:2c609fde7223
Date: 2016-11-29 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/2c609fde7223/

Log:	hg merge default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -192,6 +192,14 @@
                 assert self._finalize_.im_func is not W_Root._finalize_.im_func
             space.finalizer_queue.register_finalizer(self)
 
+    def may_unregister_rpython_finalizer(self, space):
+        """Optimization hint only: if there is no user-defined __del__()
+        method, pass the hint ``don't call any finalizer'' to rgc.
+        """
+        if not self.getclass(space).hasuserdel:
+            from rpython.rlib import rgc
+            rgc.may_ignore_finalizer(self)
+
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.pyopcode import LoopBlock, SApplicationException, Yield
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
 from pypy.interpreter.astcompiler import consts
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_uint
 
@@ -20,7 +20,7 @@
         self.running = False
         self._name = name           # may be null, use get_name()
         self._qualname = qualname   # may be null, use get_qualname()
-        if (isinstance(self, Coroutine)    # XXX would be cool not to need this
+        if (isinstance(self, Coroutine)
                 or self.pycode.co_flags & CO_YIELD_INSIDE_TRY):
             self.register_finalizer(self.space)
         self.saved_operr = None
@@ -89,7 +89,7 @@
 
         # if the frame is now marked as finished, it was RETURNed from
         if frame.frame_finished_execution:
-            self.frame = None
+            self.frame_is_finished()
             if space.is_w(w_result, space.w_None):
                 raise OperationError(space.w_StopIteration, space.w_None)
             else:
@@ -116,7 +116,7 @@
                 if e.match(space, space.w_StopIteration):
                     self._leak_stopiteration(e)
             finally:
-                self.frame = None
+                self.frame_is_finished()
             raise
         finally:
             frame.f_backref = jit.vref_None
@@ -323,6 +323,10 @@
                     break
                 block = block.previous
 
+    def frame_is_finished(self):
+        self.frame = None
+        rgc.may_ignore_finalizer(self)
+
 
 class GeneratorIterator(GeneratorOrCoroutine):
     "An iterator created by a generator."
@@ -364,7 +368,7 @@
                     break
                 # if the frame is now marked as finished, it was RETURNed from
                 if frame.frame_finished_execution:
-                    self.frame = None
+                    self.frame_is_finished()
                     break
                 results.append(w_result)     # YIELDed
         return unpack_into
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -397,7 +397,7 @@
         space = self.space
         if space.is_none(w_destructor):
             if isinstance(self, W_CDataGCP):
-                self.w_destructor = None
+                self.detach_destructor()
                 return space.w_None
             raise oefmt(space.w_TypeError,
                         "Can remove destructor only on a object "
@@ -604,6 +604,10 @@
             self.w_destructor = None
             self.space.call_function(w_destructor, self.w_original_cdata)
 
+    def detach_destructor(self):
+        self.w_destructor = None
+        self.may_unregister_rpython_finalizer(self.space)
+
 
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -55,6 +55,7 @@
         if not libhandle:
             raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed",
                         self.libname)
+        self.may_unregister_rpython_finalizer(self.ffi.space)
 
         # Clear the dict to force further accesses to do cdlopen_fetch()
         # again, and fail because the library was closed.  Note that the
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -266,6 +266,7 @@
         except SocketError:
             # cpython doesn't return any errors on close
             pass
+        self.may_unregister_rpython_finalizer(space)
 
     def connect_w(self, space, w_addr):
         """connect(address)
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -218,7 +218,7 @@
             return self.space.w_None
         return w_obj
 
-    def descr__eq__(self, space, w_ref2):
+    def compare(self, space, w_ref2, invert):
         if not isinstance(w_ref2, W_Weakref):
             return space.w_NotImplemented
         ref1 = self
@@ -226,11 +226,18 @@
         w_obj1 = ref1.dereference()
         w_obj2 = ref2.dereference()
         if w_obj1 is None or w_obj2 is None:
-            return space.is_(ref1, ref2)
-        return space.eq(w_obj1, w_obj2)
+            w_res = space.is_(ref1, ref2)
+        else:
+            w_res = space.eq(w_obj1, w_obj2)
+        if invert:
+            w_res = space.not_(w_res)
+        return w_res
+
+    def descr__eq__(self, space, w_ref2):
+        return self.compare(space, w_ref2, invert=False)
 
     def descr__ne__(self, space, w_ref2):
-        return space.not_(space.eq(self, w_ref2))
+        return self.compare(space, w_ref2, invert=True)
 
     def descr_callback(self, space):
         return self.w_callable
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -153,6 +153,14 @@
         assert not (ref1 == [])
         assert ref1 != []
 
+    def test_ne(self):
+        import _weakref
+        class X(object):
+            pass
+        ref1 = _weakref.ref(X())
+        assert ref1.__eq__(X()) is NotImplemented
+        assert ref1.__ne__(X()) is NotImplemented
+
     def test_getweakrefs(self):
         import _weakref, gc
         class A(object):
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -81,6 +81,7 @@
 
 class W_Epoll(W_Root):
     def __init__(self, space, epfd):
+        self.space = space
         self.epfd = epfd
         self.register_finalizer(space)
 
@@ -113,6 +114,7 @@
         if not self.get_closed():
             socketclose(self.epfd)
             self.epfd = -1
+            self.may_unregister_rpython_finalizer(self.space)
 
     def epoll_ctl(self, space, ctl, w_fd, eventmask, ignore_ebadf=False):
         fd = space.c_filedescriptor_w(w_fd)
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -109,6 +109,7 @@
 
 class W_Kqueue(W_Root):
     def __init__(self, space, kqfd):
+        self.space = space
         self.kqfd = kqfd
         self.register_finalizer(space)
 
@@ -137,6 +138,7 @@
             kqfd = self.kqfd
             self.kqfd = -1
             socketclose_no_errno(kqfd)
+            self.may_unregister_rpython_finalizer(self.space)
 
     def check_closed(self, space):
         if self.get_closed():
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -183,6 +183,7 @@
                 if mode == rzlib.Z_FINISH:    # release the data structures now
                     rzlib.deflateEnd(self.stream)
                     self.stream = rzlib.null_stream
+                    self.may_unregister_rpython_finalizer(space)
             finally:
                 self.unlock()
         except rzlib.RZlibError as e:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -66,15 +66,16 @@
 class W_MyType(W_MyObject):
     name = "foobar"
     flag_map_or_seq = '?'
+    hasuserdel = False
 
     def __init__(self):
         self.mro_w = [w_some_obj(), w_some_obj()]
         self.dict_w = {'__str__': w_some_obj()}
+        self.hasuserdel = True
 
     def get_module(self):
         return w_some_obj()
 
-
     def getname(self, space):
         return self.name.decode('utf-8')
 
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -155,7 +155,10 @@
 # 'old_objects_pointing_to_pinned' and doesn't have to be added again.
 GCFLAG_PINNED_OBJECT_PARENT_KNOWN = GCFLAG_PINNED
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 10    # the first unused bit
+# record that ignore_finalizer() has been called
+GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 11    # the first unused bit
 
 
 # States for the incremental GC
@@ -1672,7 +1675,7 @@
             self.rrc_minor_collection_trace()
         #
         # visit the "probably young" objects with finalizers.  They
-        # always all survive.
+        # all survive, except if IGNORE_FINALIZER is set.
         if self.probably_young_objects_with_finalizers.non_empty():
             self.deal_with_young_objects_with_finalizers()
         #
@@ -2675,6 +2678,8 @@
         while self.probably_young_objects_with_finalizers.non_empty():
             obj = self.probably_young_objects_with_finalizers.popleft()
             fq_nr = self.probably_young_objects_with_finalizers.popleft()
+            if self.header(obj).tid & GCFLAG_IGNORE_FINALIZER:
+                continue
             self.singleaddr.address[0] = obj
             self._trace_drag_out1(self.singleaddr)
             obj = self.singleaddr.address[0]
@@ -2697,6 +2702,8 @@
             fq_nr = self.old_objects_with_finalizers.popleft()
             ll_assert(self._finalization_state(x) != 1,
                       "bad finalization state 1")
+            if self.header(x).tid & GCFLAG_IGNORE_FINALIZER:
+                continue
             if self.header(x).tid & GCFLAG_VISITED:
                 new_with_finalizer.append(x)
                 new_with_finalizer.append(fq_nr)
@@ -2787,6 +2794,9 @@
         self.objects_to_trace.append(obj)
         self.visit_all_objects()
 
+    def ignore_finalizer(self, obj):
+        self.header(obj).tid |= GCFLAG_IGNORE_FINALIZER
+
 
     # ----------
     # Weakrefs
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -545,6 +545,12 @@
                                              s_gcref],
                                             annmodel.s_None)
 
+        self.ignore_finalizer_ptr = None
+        if hasattr(GCClass, 'ignore_finalizer'):
+            self.ignore_finalizer_ptr = getfn(GCClass.ignore_finalizer,
+                                              [s_gc, SomeAddress()],
+                                              annmodel.s_None)
+
     def create_custom_trace_funcs(self, gc, rtyper):
         custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
         rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1572,6 +1578,13 @@
         hop.genop("cast_adr_to_ptr", [v_adr],
                   resultvar = hop.spaceop.result)
 
+    def gct_gc_ignore_finalizer(self, hop):
+        if self.ignore_finalizer_ptr is not None:
+            v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+                              resulttype=llmemory.Address)
+            hop.genop("direct_call", [self.ignore_finalizer_ptr,
+                                      self.c_const_gc, v_adr])
+
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
 
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -4,7 +4,7 @@
 """
 
 import os, stat, errno, sys
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rgc
 from rpython.rlib.objectmodel import enforceargs
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder
@@ -294,6 +294,7 @@
         if ll_file:
             # double close is allowed
             self._ll_file = lltype.nullptr(FILEP.TO)
+            rgc.may_ignore_finalizer(self)
             do_close = self._close2[0]
             try:
                 if do_close:
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -527,6 +527,13 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
+ at jit.dont_look_inside
+def may_ignore_finalizer(obj):
+    """Optimization hint: says that it is valid for any finalizer
+    for 'obj' to be ignored, depending on the GC."""
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    llop.gc_ignore_finalizer(lltype.Void, obj)
+
 
 # ____________________________________________________________
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -486,6 +486,7 @@
     'gc_add_memory_pressure': LLOp(),
     'gc_fq_next_dead'     : LLOp(),
     'gc_fq_register'      : LLOp(),
+    'gc_ignore_finalizer' : LLOp(canrun=True),
 
     'gc_rawrefcount_init':              LLOp(),
     'gc_rawrefcount_create_link_pypy':  LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -736,6 +736,9 @@
     assert isinstance(x, bool)
     return x
 
+def op_gc_ignore_finalizer(obj):
+    pass
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/rpython/translator/c/src/mem.h b/rpython/translator/c/src/mem.h
--- a/rpython/translator/c/src/mem.h
+++ b/rpython/translator/c/src/mem.h
@@ -152,6 +152,7 @@
 #define OP_GC_IS_RPY_INSTANCE(x, r)      r = 0
 #define OP_GC_DUMP_RPY_HEAP(fd, r)       r = 0
 #define OP_GC_SET_EXTRA_THRESHOLD(x, r)  /* nothing */
+#define OP_GC_IGNORE_FINALIZER(x, r)     /* nothing */
 
 /****************************/
 /* The "asmgcc" root finder */
diff --git a/rpython/translator/c/test/test_boehm.py b/rpython/translator/c/test/test_boehm.py
--- a/rpython/translator/c/test/test_boehm.py
+++ b/rpython/translator/c/test/test_boehm.py
@@ -409,7 +409,9 @@
         #
         def fn():
             for i in range(1000):
-                fq.register_finalizer(A(i))
+                x = A(i)
+                fq.register_finalizer(x)
+                rgc.may_ignore_finalizer(x)   # this is ignored with Boehm
             rgc.collect()
             rgc.collect()
             if glob.triggered == 0:
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1705,6 +1705,38 @@
             res = self.run("limited_memory_linux", -1, runner=myrunner)
             assert res == 42
 
+    def define_ignore_finalizer(cls):
+        class X(object):
+            pass
+        class FQ(rgc.FinalizerQueue):
+            Class = X
+            def finalizer_trigger(self):
+                pass
+        queue = FQ()
+        def g():
+            x1 = X()
+            x2 = X()
+            queue.register_finalizer(x1)
+            queue.register_finalizer(x2)
+            rgc.may_ignore_finalizer(x1)
+        g._dont_inline_ = True
+        def f():
+            g()
+            rgc.collect()
+            seen = 0
+            while True:
+                obj = queue.next_dead()
+                if obj is None:
+                    break
+                seen += 1
+            return seen
+        assert f() == 2    # untranslated: may_ignore_finalizer() is ignored
+        return f
+
+    def test_ignore_finalizer(self):
+        res = self.run("ignore_finalizer")
+        assert res == 1    # translated: x1 is removed from the list
+
 
 # ____________________________________________________________________
 

From pypy.commits at gmail.com  Tue Nov 29 10:35:15 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:35:15 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583da033.e7b1c20a.de63f.fce1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88746:f71dea9e2218
Date: 2016-11-29 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/f71dea9e2218/

Log:	fix test

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -373,12 +373,12 @@
         import _socket
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
         try:
-            expected = ('<socket object, fd=%s, family=%s, type=%s, protocol=%s>'
+            expected = ('<socket object, fd=%s, family=%s, type=%s, proto=%s>'
                         % (s.fileno(), s.family, s.type, s.proto))
             assert repr(s) == expected
         finally:
             s.close()
-        expected = ('<socket object, fd=-1, family=%s, type=%s, protocol=%s>'
+        expected = ('<socket object, fd=-1, family=%s, type=%s, proto=%s>'
                     % (s.family, s.type, s.proto))
         assert repr(s) == expected
 

From pypy.commits at gmail.com  Tue Nov 29 10:42:19 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:42:19 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: optimize away Coroutine._finalize_()
 earlier
Message-ID: <583da1db.8c1f1c0a.30a1a.1ed8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88747:c8d9cf08602f
Date: 2016-11-29 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/c8d9cf08602f/

Log:	optimize away Coroutine._finalize_() earlier

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -107,6 +107,14 @@
         if self.saved_operr is not None:
             ec.set_sys_exc_info(self.saved_operr)
             self.saved_operr = None
+        #
+        # Optimization only: after we've started a Coroutine without
+        # CO_YIELD_INSIDE_TRY, then Coroutine._finalize_() will be a no-op
+        if (isinstance(self, Coroutine)
+                and frame.last_instr == -1
+                and not (self.pycode.co_flags & CO_YIELD_INSIDE_TRY)):
+            rgc.may_ignore_finalizer(self)
+        #
         self.running = True
         try:
             w_result = frame.execute_frame(self, w_arg_or_err)

From pypy.commits at gmail.com  Tue Nov 29 10:47:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:47:23 -0800 (PST)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <583da30b.6249c20a.b67cc.1864@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88748:1016e6fc4bf6
Date: 2016-11-29 16:46 +0100
http://bitbucket.org/pypy/pypy/changeset/1016e6fc4bf6/

Log:	translation fix

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -528,6 +528,7 @@
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
 @jit.dont_look_inside
+ at specialize.argtype(0)
 def may_ignore_finalizer(obj):
     """Optimization hint: says that it is valid for any finalizer
     for 'obj' to be ignored, depending on the GC."""

From pypy.commits at gmail.com  Tue Nov 29 10:47:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 07:47:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <583da30d.8a29c20a.df6a.0df8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88749:179f6db6f15b
Date: 2016-11-29 16:46 +0100
http://bitbucket.org/pypy/pypy/changeset/179f6db6f15b/

Log:	hg merge default

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -528,6 +528,7 @@
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
 @jit.dont_look_inside
+ at specialize.argtype(0)
 def may_ignore_finalizer(obj):
     """Optimization hint: says that it is valid for any finalizer
     for 'obj' to be ignored, depending on the GC."""

From pypy.commits at gmail.com  Tue Nov 29 11:33:00 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 08:33:00 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: only Exceptions (not BaseExceptions) are
 ignored when calling module_repr
Message-ID: <583dadbc.d32f1c0a.37189.3f03@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88750:10fde59504a2
Date: 2016-11-29 17:32 +0100
http://bitbucket.org/pypy/pypy/changeset/10fde59504a2/

Log:	only Exceptions (not BaseExceptions) are ignored when calling
	module_repr

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -125,8 +125,9 @@
         if w_loader is not None:
             try:
                 return space.call_method(w_loader, "module_repr", self)
-            except OperationError:
-                pass
+            except OperationError, operr:
+                if not operr.match(space, space.w_Exception):
+                    raise
         try:
             w_name = space.getattr(self, space.wrap('__name__'))
             name = space.unicode_w(space.repr(w_name))
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -129,6 +129,20 @@
         expected_repr = "<module 'test_module' ({})>".format(loader_repr)
         assert mod_repr == expected_repr
 
+    def test_repr_with_loader_with_raising_module_repr2(self):
+        import sys
+        test_module = type(sys)("test_module", "doc")
+        # If an exception occurs in module_repr(), the exception is caught
+        # and discarded, and the calculation of the module&#8217;s repr continues
+        # as if module_repr() did not exist.
+        class CustomLoaderWithRaisingRepr:
+            @classmethod
+            def module_repr(cls, module):
+                raise KeyboardInterrupt
+
+        test_module.__loader__ = CustomLoaderWithRaisingRepr
+        raises(KeyboardInterrupt, 'repr(test_module)')
+
     def test_repr_with_raising_loader_and___file__(self):
         import sys
         test_module = type(sys)("test_module", "doc")

From pypy.commits at gmail.com  Tue Nov 29 11:46:06 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 08:46:06 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: just call importlib._module_repr directly
Message-ID: <583db0ce.88711c0a.8225f.43d7@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r88751:1b3860266dbd
Date: 2016-11-29 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/1b3860266dbd/

Log:	just call importlib._module_repr directly

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -121,32 +121,8 @@
         return space.newtuple(tup_return)
 
     def descr_module__repr__(self, space):
-        w_loader = space.finditem(self.w_dict, space.wrap('__loader__'))
-        if w_loader is not None:
-            try:
-                return space.call_method(w_loader, "module_repr", self)
-            except OperationError, operr:
-                if not operr.match(space, space.w_Exception):
-                    raise
-        try:
-            w_name = space.getattr(self, space.wrap('__name__'))
-            name = space.unicode_w(space.repr(w_name))
-        except OperationError:
-            name = u"'?'"
-
-        try:
-            w___file__ = space.getattr(self, space.wrap('__file__'))
-        except OperationError:
-            w___file__ = space.w_None
-        if not space.isinstance_w(w___file__, space.w_unicode):
-            if w_loader is not None:
-                w_loader_repr = space.unicode_w(space.repr(w_loader))
-                return space.wrap(u"<module %s (%s)>" % (name, w_loader_repr))
-            else:
-                return space.wrap(u"<module %s>" % (name,))
-        else:
-            __file__ = space.unicode_w(space.repr(w___file__))
-            return space.wrap(u"<module %s from %s>" % (name, __file__))
+        w_importlib = space.getbuiltinmodule('_frozen_importlib')
+        return space.call_method(w_importlib, "_module_repr", self)
 
     def descr_getattribute(self, space, w_attr):
         from pypy.objspace.descroperation import object_getattribute

From pypy.commits at gmail.com  Tue Nov 29 11:53:54 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 08:53:54 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: random attempts at fixing tests
Message-ID: <583db2a2.0b561c0a.37b32.439b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88752:d2ee95db07b2
Date: 2016-11-29 17:33 +0100
http://bitbucket.org/pypy/pypy/changeset/d2ee95db07b2/

Log:	random attempts at fixing tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -272,11 +272,11 @@
             _write = libc.load_function(BWrite, 'write')
             i = 0
             fd0, fd1 = os.pipe()
-            buffer = _cffi_backend.newp(BCharP, 'A')
+            buffer = _cffi_backend.newp(BCharP, b'A')
             while i < 300:
                 tmp = _write(fd1, buffer, 1)   # ID: cfficall
                 assert tmp == 1
-                assert os.read(fd0, 2) == 'A'
+                assert os.read(fd0, 2) == b'A'
                 i += 1
             os.close(fd0)
             os.close(fd1)
@@ -410,7 +410,7 @@
         i161 = int_lt(i160, i43)
         guard_true(i161, descr=...)
         i162 = int_add(i160, 1)
-        setfield_gc(p22, i162, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current .>)
+        setfield_gc(p22, i162, descr=<FieldS pypy.module.__builtin__.functional.W_IntRangeIterator.inst_current .>)
         guard_not_invalidated(descr=...)
         p163 = force_token()
         p164 = force_token()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py
--- a/pypy/module/pypyjit/test_pypy_c/test_globals.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py
@@ -16,5 +16,6 @@
         assert log.result == 500
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id("loadglobal", """
+            p1 = getfield_gc_r(..., descr=...)     # dead
             guard_not_invalidated(descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py
--- a/pypy/module/pypyjit/test_pypy_c/test_shift.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py
@@ -196,8 +196,7 @@
         """
         from sys import maxint
 
-        def main(a, b, c):
-            from sys import maxint
+        def main(a, b, c, maxint):
             i = sa = 0
             while i < 300:
                 if 0 < a < 10: pass
@@ -215,4 +214,4 @@
         for a in (1, 4, 8, 100):
             for b in (-10, 10, -201, 201, -maxint/3, maxint/3):
                 for c in (-10, 10, -maxint/3, maxint/3):
-                    yield self.run_and_check, main, [a, b, c]
+                    yield self.run_and_check, main, [a, b, c, maxint]

From pypy.commits at gmail.com  Tue Nov 29 11:53:56 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 08:53:56 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: random attempts at getting more info
Message-ID: <583db2a4.2a0bc30a.71fd5.be5e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88753:e29cc6279dfb
Date: 2016-11-29 17:53 +0100
http://bitbucket.org/pypy/pypy/changeset/e29cc6279dfb/

Log:	random attempts at getting more info

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -7,12 +7,12 @@
 class TestString(BaseTestPyPyC):
     def test_lookup_default_encoding(self):
         def main(n):
-            import string
             i = 0
-            letters = string.letters
-            uletters = unicode(string.letters)
+            letters = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+            uletters = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
             while i < n:
-                i += letters[i % len(letters)] == uletters[i % len(letters)]
+                i += (letters[i % len(letters)].decode() ==
+                      uletters[i % len(letters)])
             return i
 
         log = self.run(main, [300], import_site=True)
@@ -45,7 +45,7 @@
 
     def test_long(self):
         def main(n):
-            import string
+            digits = '0123456789'
             i = 1
             while i < n:
                 i += int(long(string.digits[i % len(string.digits)], 16))

From pypy.commits at gmail.com  Tue Nov 29 11:53:58 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 08:53:58 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583db2a6.0a4cc20a.25f32.2907@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88754:49831b800735
Date: 2016-11-29 17:53 +0100
http://bitbucket.org/pypy/pypy/changeset/49831b800735/

Log:	merge heads

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -121,31 +121,8 @@
         return space.newtuple(tup_return)
 
     def descr_module__repr__(self, space):
-        w_loader = space.finditem(self.w_dict, space.wrap('__loader__'))
-        if w_loader is not None:
-            try:
-                return space.call_method(w_loader, "module_repr", self)
-            except OperationError:
-                pass
-        try:
-            w_name = space.getattr(self, space.wrap('__name__'))
-            name = space.unicode_w(space.repr(w_name))
-        except OperationError:
-            name = u"'?'"
-
-        try:
-            w___file__ = space.getattr(self, space.wrap('__file__'))
-        except OperationError:
-            w___file__ = space.w_None
-        if not space.isinstance_w(w___file__, space.w_unicode):
-            if w_loader is not None:
-                w_loader_repr = space.unicode_w(space.repr(w_loader))
-                return space.wrap(u"<module %s (%s)>" % (name, w_loader_repr))
-            else:
-                return space.wrap(u"<module %s>" % (name,))
-        else:
-            __file__ = space.unicode_w(space.repr(w___file__))
-            return space.wrap(u"<module %s from %s>" % (name, __file__))
+        w_importlib = space.getbuiltinmodule('_frozen_importlib')
+        return space.call_method(w_importlib, "_module_repr", self)
 
     def descr_getattribute(self, space, w_attr):
         from pypy.objspace.descroperation import object_getattribute
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -129,6 +129,20 @@
         expected_repr = "<module 'test_module' ({})>".format(loader_repr)
         assert mod_repr == expected_repr
 
+    def test_repr_with_loader_with_raising_module_repr2(self):
+        import sys
+        test_module = type(sys)("test_module", "doc")
+        # If an exception occurs in module_repr(), the exception is caught
+        # and discarded, and the calculation of the module&#8217;s repr continues
+        # as if module_repr() did not exist.
+        class CustomLoaderWithRaisingRepr:
+            @classmethod
+            def module_repr(cls, module):
+                raise KeyboardInterrupt
+
+        test_module.__loader__ = CustomLoaderWithRaisingRepr
+        raises(KeyboardInterrupt, 'repr(test_module)')
+
     def test_repr_with_raising_loader_and___file__(self):
         import sys
         test_module = type(sys)("test_module", "doc")

From pypy.commits at gmail.com  Tue Nov 29 11:55:50 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 Nov 2016 08:55:50 -0800 (PST)
Subject: [pypy-commit] pypy default: unspotted copy-paste mistake in
 descroperation, fix error message if __oct__, 
Message-ID: <583db316.e7b1c20a.de63f.23fb@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r88755:e30a24ad4ccd
Date: 2016-11-29 17:54 +0100
http://bitbucket.org/pypy/pypy/changeset/e30a24ad4ccd/

Log:	unspotted copy-paste mistake in descroperation, fix error message if
	__oct__, __repr__, __hex__ return a non-string

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -853,7 +853,7 @@
                 if not e.match(space, space.w_TypeError):
                     raise
                 raise oefmt(space.w_TypeError,
-                            "%(specialname)s returned non-%(targetname)s "
+                            "%(specialname)s returned non-string "
                             "(type '%%T')", w_result)
             else:
                 # re-wrap the result as a real string
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -315,7 +315,8 @@
                 assert operate(A()) == "world" * n
             assert type(operate(A())) is str
             answer = 42
-            raises(TypeError, operate, A())
+            excinfo = raises(TypeError, operate, A())
+            assert "returned non-string (type 'int')" in str(excinfo.value)
 
     def test_missing_getattribute(self):
         class X(object):

From pypy.commits at gmail.com  Tue Nov 29 12:05:04 2016
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 29 Nov 2016 09:05:04 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: add two entries to planning/py3.5,
 remove me from the lock section
Message-ID: <583db540.e626c20a.f98d5.4ff5@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: extradoc
Changeset: r5754:232b87cd46d7
Date: 2016-11-29 18:04 +0100
http://bitbucket.org/pypy/extradoc/changeset/232b87cd46d7/

Log:	add two entries to planning/py3.5, remove me from the lock section

diff --git a/planning/py3.5/milestone-1-progress.rst b/planning/py3.5/milestone-1-progress.rst
--- a/planning/py3.5/milestone-1-progress.rst
+++ b/planning/py3.5/milestone-1-progress.rst
@@ -5,8 +5,6 @@
 In-progress ("Lock" section)
 ----------------------------
 
-* (richard) multiprocessing module changes
-
 
 Misc stuff not formally in any milestone
 ----------------------------------------
@@ -47,6 +45,12 @@
 * bonus: use all the features of _pypyjson from the json module again
   (eg c_encode_basestring_ascii)
 
+* socket get lots of new methods (e.g. recvmsg, ...), all test stdlib tests
+  are now skipping them
+
+* _hashlib pbkdf2_hmac has a new 'fast' implemention in cpython,
+  unsure if we are eager to implement that right now
+
 
 Milestone 1 (Aug-Sep-Oct 2016)
 ------------------------------

From pypy.commits at gmail.com  Tue Nov 29 17:07:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 14:07:23 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <583dfc1b.55911c0a.75cd.bebd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r823:921754465131
Date: 2016-11-29 23:07 +0100
http://bitbucket.org/pypy/pypy.org/changeset/921754465131/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 63.1
+      value: 63.2
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66267 of $105000 (63.1%)
+   $66339 of $105000 (63.2%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2128 left
+    we have $2192 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Tue Nov 29 18:51:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Tue, 29 Nov 2016 15:51:16 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: Mention RevDB from here
Message-ID: <583e1474.031f1c0a.f539c.d937@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r824:0917de0cf92d
Date: 2016-11-30 00:50 +0100
http://bitbucket.org/pypy/pypy.org/changeset/0917de0cf92d/

Log:	Mention RevDB from here

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -182,6 +182,11 @@
 <li>The most up-to-date <a class="reference external" href="http://buildbot.pypy.org/nightly/trunk/">nightly binary builds</a> with a JIT, if the official
 release is too old for what you want to do. There are versions for
 different libc on this site too.</li>
+<li>Reverse debugger (RevDB): This version enables debugging your Python
+programs by going forward and backward in time.  See the <a class="reference external" href="https://bitbucket.org/pypy/revdb/">separate
+documentation</a>.</li>
+</ul>
+<ul class="simple">
 <li>Sandboxing: A special safe version.  Read the docs about <a class="reference external" href="features.html#sandboxing">sandboxing</a>.
 (It is also possible to <a class="reference internal" href="#translate">translate</a> a version that includes both
 sandboxing and the JIT compiler, although as the JIT is relatively
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -189,6 +189,12 @@
   release is too old for what you want to do. There are versions for
   different libc on this site too.
 
+* Reverse debugger (RevDB): This version enables debugging your Python
+  programs by going forward and backward in time.  See the `separate
+  documentation`__.
+
+.. __: https://bitbucket.org/pypy/revdb/
+
 * Sandboxing: A special safe version.  Read the docs about sandboxing_.
   (It is also possible to translate_ a version that includes both
   sandboxing and the JIT compiler, although as the JIT is relatively

From pypy.commits at gmail.com  Wed Nov 30 04:46:43 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 01:46:43 -0800 (PST)
Subject: [pypy-commit] extradoc extradoc: Yet another weakdict issue
Message-ID: <583ea003.e2acc20a.38e57.74dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r5755:b2c2c758f9a7
Date: 2016-11-30 10:46 +0100
http://bitbucket.org/pypy/extradoc/changeset/b2c2c758f9a7/

Log:	Yet another weakdict issue

diff --git a/planning/py3.5/cpython-crashers.rst b/planning/py3.5/cpython-crashers.rst
--- a/planning/py3.5/cpython-crashers.rst
+++ b/planning/py3.5/cpython-crashers.rst
@@ -172,6 +172,38 @@
 * _collectionsmodule.c: deque_repr uses "[...]" as repr if recursion is
   detected.  I'd suggest that "deque(...)" is clearer---it's not a list.
 
+* weak dicts (both kinds) and weak sets have an implementation of
+  __len__ which doesn't give the "expected" result on PyPy, and in some
+  cases on CPython too.  I'm not sure what is expected and what is not.
+  Here is an example on CPython 3.5.2+ (using a thread to run the weakref
+  callbacks only, not to explicitly inspect or modify 'd')::
+
+    import weakref, _thread
+    from queue import Queue
+
+    queue = Queue()
+    def subthread(queue):
+        while True:
+            queue.get()
+    _thread.start_new_thread(subthread, (queue,))
+
+    class X:
+        pass
+    d = weakref.WeakValueDictionary()
+    while True:
+        x = X()
+        d[52] = x
+        queue.put(x)
+        del x
+        while list(d) != []:
+            pass
+        assert len(d) == 0  # we've checked that list(d)==[], but this may fail
+
+  On CPython I've seen the assert fail only after editing the function
+  WeakValueDictionary.__init__.remove() to add ``time.sleep(0.01)`` as
+  the first line.  Otherwise I guess the timings happen to make that test
+  pass.
+
 
 Other issues of "dubious IMHO" status
 -------------------------------------

From pypy.commits at gmail.com  Wed Nov 30 04:55:44 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 01:55:44 -0800 (PST)
Subject: [pypy-commit] pypy.org extradoc: tweak
Message-ID: <583ea220.2aa9c20a.88c0a.5605@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r825:c22e43911290
Date: 2016-11-30 10:55 +0100
http://bitbucket.org/pypy/pypy.org/changeset/c22e43911290/

Log:	tweak

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -182,8 +182,8 @@
 <li>The most up-to-date <a class="reference external" href="http://buildbot.pypy.org/nightly/trunk/">nightly binary builds</a> with a JIT, if the official
 release is too old for what you want to do. There are versions for
 different libc on this site too.</li>
-<li>Reverse debugger (RevDB): This version enables debugging your Python
-programs by going forward and backward in time.  See the <a class="reference external" href="https://bitbucket.org/pypy/revdb/">separate
+<li>Reverse debugger: This version enables debugging your Python
+programs by going forward and backward in time.  See the <a class="reference external" href="https://bitbucket.org/pypy/revdb/">RevDB
 documentation</a>.</li>
 </ul>
 <ul class="simple">
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -189,8 +189,8 @@
   release is too old for what you want to do. There are versions for
   different libc on this site too.
 
-* Reverse debugger (RevDB): This version enables debugging your Python
-  programs by going forward and backward in time.  See the `separate
+* Reverse debugger: This version enables debugging your Python
+  programs by going forward and backward in time.  See the `RevDB
   documentation`__.
 
 .. __: https://bitbucket.org/pypy/revdb/

From pypy.commits at gmail.com  Wed Nov 30 05:11:24 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 02:11:24 -0800 (PST)
Subject: [pypy-commit] pypy default: oops, fix the GC
Message-ID: <583ea5cc.448e1c0a.758f6.7d2e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88756:fe98eb698187
Date: 2016-11-30 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/fe98eb698187/

Log:	oops, fix the GC

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1164,6 +1164,11 @@
             obj = self.get_forwarding_address(obj)
         return self.get_type_id(obj)
 
+    def get_possibly_forwarded_tid(self, obj):
+        if self.is_in_nursery(obj) and self.is_forwarded(obj):
+            obj = self.get_forwarding_address(obj)
+        return self.header(obj).tid
+
     def get_total_memory_used(self):
         """Return the total memory used, not counting any object in the
         nursery: only objects in the ArenaCollection or raw-malloced.
@@ -2678,7 +2683,7 @@
         while self.probably_young_objects_with_finalizers.non_empty():
             obj = self.probably_young_objects_with_finalizers.popleft()
             fq_nr = self.probably_young_objects_with_finalizers.popleft()
-            if self.header(obj).tid & GCFLAG_IGNORE_FINALIZER:
+            if self.get_possibly_forwarded_tid(obj) & GCFLAG_IGNORE_FINALIZER:
                 continue
             self.singleaddr.address[0] = obj
             self._trace_drag_out1(self.singleaddr)

From pypy.commits at gmail.com  Wed Nov 30 05:15:02 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 02:15:02 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <583ea6a6.0a22c20a.3594d.7a32@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88757:fbe001ae02e9
Date: 2016-11-30 11:14 +0100
http://bitbucket.org/pypy/pypy/changeset/fbe001ae02e9/

Log:	hg merge default

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -705,7 +705,7 @@
                 return w_result
 
             raise oefmt(space.w_TypeError,
-                        "%(specialname)s returned non-%(targetname)s (type "
+                        "%(specialname)s returned non-string (type "
                         "'%%T')", w_result)
         assert not hasattr(DescrOperation, %(targetname)r)
         DescrOperation.%(targetname)s = %(targetname)s
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -280,7 +280,8 @@
             assert operate(A()) == "hello" * n
             assert type(operate(A())) is str
             answer = 42
-            raises(TypeError, operate, A())
+            excinfo = raises(TypeError, operate, A())
+            assert "returned non-string (type 'int')" in str(excinfo.value)
 
     def test_string_results_unicode(self):
         class A(object):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1164,6 +1164,11 @@
             obj = self.get_forwarding_address(obj)
         return self.get_type_id(obj)
 
+    def get_possibly_forwarded_tid(self, obj):
+        if self.is_in_nursery(obj) and self.is_forwarded(obj):
+            obj = self.get_forwarding_address(obj)
+        return self.header(obj).tid
+
     def get_total_memory_used(self):
         """Return the total memory used, not counting any object in the
         nursery: only objects in the ArenaCollection or raw-malloced.
@@ -2678,7 +2683,7 @@
         while self.probably_young_objects_with_finalizers.non_empty():
             obj = self.probably_young_objects_with_finalizers.popleft()
             fq_nr = self.probably_young_objects_with_finalizers.popleft()
-            if self.header(obj).tid & GCFLAG_IGNORE_FINALIZER:
+            if self.get_possibly_forwarded_tid(obj) & GCFLAG_IGNORE_FINALIZER:
                 continue
             self.singleaddr.address[0] = obj
             self._trace_drag_out1(self.singleaddr)

From pypy.commits at gmail.com  Wed Nov 30 06:00:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 03:00:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583eb14a.820bc30a.f79f4.746f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88758:c8256e8d14ee
Date: 2016-11-30 11:59 +0100
http://bitbucket.org/pypy/pypy/changeset/c8256e8d14ee/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_shift.py b/pypy/module/pypyjit/test_pypy_c/test_shift.py
--- a/pypy/module/pypyjit/test_pypy_c/test_shift.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_shift.py
@@ -209,7 +209,7 @@
                 sa += (b<<100)>>100
                 sa += (c<<100)>>100
                 i += 1
-            return long(sa)
+            return sa
 
         for a in (1, 4, 8, 100):
             for b in (-10, 10, -201, 201, -maxint/3, maxint/3):

From pypy.commits at gmail.com  Wed Nov 30 06:16:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 03:16:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Painfully fix the test so that it tests
 something reasonable on Python 3 (not passing yet)
Message-ID: <583eb4f7.820bc30a.f79f4.7af4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88759:d84373506b5d
Date: 2016-11-30 12:15 +0100
http://bitbucket.org/pypy/pypy/changeset/d84373506b5d/

Log:	Painfully fix the test so that it tests something reasonable on
	Python 3 (not passing yet)

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -9,10 +9,11 @@
         def main(n):
             i = 0
             letters = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+            letters = [b'%c' % n for n in letters]  # list of single-char bytes
             uletters = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
             while i < n:
-                i += (letters[i % len(letters)].decode() ==
-                      uletters[i % len(letters)])
+                c = letters[i % len(uletters)]
+                i += (c.decode() == uletters[i % len(uletters)])
             return i
 
         log = self.run(main, [300], import_site=True)

From pypy.commits at gmail.com  Wed Nov 30 08:20:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 05:20:08 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Add another failing test about
 'bytes([i])'
Message-ID: <583ed208.aaa3c20a.7ac19.b697@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88760:65618280cf6c
Date: 2016-11-30 14:19 +0100
http://bitbucket.org/pypy/pypy/changeset/65618280cf6c/

Log:	Add another failing test about 'bytes([i])'

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -5,14 +5,35 @@
 
 
 class TestString(BaseTestPyPyC):
+
+    def test_python3_missing_bchr(self):
+        # Check that 'bytes([i])' is special-cased into something
+        # efficient, as Python 3.5 doesn't have a bchr() function or
+        # anything more direct.
+        def main(n):
+            i = 0
+            result = b''
+            while i < n:
+                c = bytes([i])
+                result += c
+                i += 1
+            return i
+        log = self.run(main, [255])
+        assert log.result == 255
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            #...
+            --TICK--
+            jump(..., descr=...)
+        """)
+
     def test_lookup_default_encoding(self):
         def main(n):
             i = 0
             letters = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-            letters = [b'%c' % n for n in letters]  # list of single-char bytes
             uletters = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
             while i < n:
-                c = letters[i % len(uletters)]
+                c = bytes([letters[i % len(uletters)]])
                 i += (c.decode() == uletters[i % len(uletters)])
             return i
 

From pypy.commits at gmail.com  Wed Nov 30 08:42:13 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 05:42:13 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Don't crash on a pypy3 without cpyext
Message-ID: <583ed735.6249c20a.b67cc.c4e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88761:c7dc55d733de
Date: 2016-11-25 14:16 +0000
http://bitbucket.org/pypy/pypy/changeset/c7dc55d733de/

Log:	Don't crash on a pypy3 without cpyext

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -60,6 +60,8 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
+    so_list = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    so_ext = (so_list or ['.so'])[0]
     g = {}
     g['CC'] = "gcc -pthread"
     g['CXX'] = "g++ -pthread"
@@ -67,7 +69,7 @@
     g['CFLAGS'] = "-DNDEBUG -O2"
     g['CCSHARED'] = "-fPIC"
     g['LDSHARED'] = "gcc -pthread -shared"
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['SO'] = so_ext
     g['SHLIB_SUFFIX'] = g['SO']
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"

From pypy.commits at gmail.com  Wed Nov 30 08:42:16 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 05:42:16 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583ed738.272cc20a.647e8.baf3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88762:14fd276cb8b4
Date: 2016-11-30 13:51 +0000
http://bitbucket.org/pypy/pypy/changeset/14fd276cb8b4/

Log:	merge heads

diff too long, truncating to 2000 out of 16891 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -74,6 +74,7 @@
 ^rpython/doc/_build/.*$
 ^compiled
 ^.git/
+.git/
 ^.hypothesis/
 ^release/
 ^rpython/_cache$
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -74,6 +74,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
@@ -109,6 +122,12 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
     if args:
         vals = []
         for name in args:
diff --git a/lib-python/3/_weakrefset.py b/lib-python/3/_weakrefset.py
--- a/lib-python/3/_weakrefset.py
+++ b/lib-python/3/_weakrefset.py
@@ -65,7 +65,14 @@
                     yield item
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, item):
         try:
diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -370,12 +370,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -428,15 +429,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py
--- a/lib-python/3/ssl.py
+++ b/lib-python/3/ssl.py
@@ -145,6 +145,7 @@
 from socket import SOL_SOCKET, SO_TYPE
 import base64        # for DER-to-PEM translation
 import errno
+import warnings
 
 
 socket_error = OSError  # keep that public name in module namespace
@@ -405,12 +406,16 @@
 
     def _load_windows_store_certs(self, storename, purpose):
         certs = bytearray()
-        for cert, encoding, trust in enum_certificates(storename):
-            # CA certs are never PKCS#7 encoded
-            if encoding == "x509_asn":
-                if trust is True or purpose.oid in trust:
-                    certs.extend(cert)
-        self.load_verify_locations(cadata=certs)
+        try:
+            for cert, encoding, trust in enum_certificates(storename):
+                # CA certs are never PKCS#7 encoded
+                if encoding == "x509_asn":
+                    if trust is True or purpose.oid in trust:
+                        certs.extend(cert)
+        except PermissionError:
+            warnings.warn("unable to enumerate Windows certificate store")
+        if certs:
+            self.load_verify_locations(cadata=certs)
         return certs
 
     def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
@@ -560,7 +565,7 @@
         server hostame is set."""
         return self._sslobj.server_hostname
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to 'len' bytes from the SSL object and return them.
 
         If 'buffer' is provided, read into this buffer and return the number of
@@ -569,7 +574,7 @@
         if buffer is not None:
             v = self._sslobj.read(len, buffer)
         else:
-            v = self._sslobj.read(len or 1024)
+            v = self._sslobj.read(len)
         return v
 
     def write(self, data):
@@ -745,7 +750,8 @@
                         # non-blocking
                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
                     self.do_handshake()
-            except:
+
+            except (OSError, ValueError):
                 self.close()
                 raise
 
@@ -774,7 +780,7 @@
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
 
diff --git a/lib-python/3/test/capath/0e4015b9.0 b/lib-python/3/test/capath/0e4015b9.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/0e4015b9.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/capath/ce7b8643.0 b/lib-python/3/test/capath/ce7b8643.0
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/capath/ce7b8643.0
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
+bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
+A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo
+b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0
+aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
+Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
+Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/test/selfsigned_pythontestdotnet.pem b/lib-python/3/test/selfsigned_pythontestdotnet.pem
--- a/lib-python/3/test/selfsigned_pythontestdotnet.pem
+++ b/lib-python/3/test/selfsigned_pythontestdotnet.pem
@@ -1,5 +1,5 @@
 -----BEGIN CERTIFICATE-----
-MIIChzCCAfCgAwIBAgIJAKGU95wKR8pSMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
+MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV
 BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
 IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv
 bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG
@@ -8,9 +8,9 @@
 aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ
 Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm
 Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv
-EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjKTAnMCUGA1UdEQQeMByCGnNl
-bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MA0GCSqGSIb3DQEBBQUAA4GBAIOXmdtM
-eG9qzP9TiXW/Gc/zI4cBfdCpC+Y4gOfC9bQUC7hefix4iO3+iZjgy3X/FaRxUUoV
-HKiXcXIaWqTSUWp45cSh0MbwZXudp6JIAptzdAhvvCrPKeC9i9GvxsPD4LtDAL97
-vSaxQBezA7hdxZd90/EeyMgVZgAnTCnvAWX9
+EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl
+bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN
+AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h
+TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515
+C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM=
 -----END CERTIFICATE-----
diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -510,6 +510,7 @@
             fut.add_done_callback(lambda *args: self.loop.stop())
             self.loop.run_forever()
             fut = None # Trigger Future.__del__ or futures._TracebackLogger
+            support.gc_collect()
             if PY34:
                 # Future.__del__ in Python 3.4 logs error with
                 # an actual exception context
diff --git a/lib-python/3/test/test_asyncio/test_futures.py b/lib-python/3/test/test_asyncio/test_futures.py
--- a/lib-python/3/test/test_asyncio/test_futures.py
+++ b/lib-python/3/test/test_asyncio/test_futures.py
@@ -238,6 +238,7 @@
         fut.set_exception(RuntimeError('boom'))
         del fut
         test_utils.run_briefly(self.loop)
+        support.gc_collect()
         self.assertTrue(m_log.error.called)
 
     @mock.patch('asyncio.base_events.logger')
diff --git a/lib-python/3/test/test_complex.py b/lib-python/3/test/test_complex.py
--- a/lib-python/3/test/test_complex.py
+++ b/lib-python/3/test/test_complex.py
@@ -310,7 +310,7 @@
         self.assertRaises(TypeError, float, 5+3j)
         self.assertRaises(ValueError, complex, "")
         self.assertRaises(TypeError, complex, None)
-        self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None)
+        self.assertRaisesRegex(TypeError, " 'NoneType'", complex, None)
         self.assertRaises(ValueError, complex, "\0")
         self.assertRaises(ValueError, complex, "3\09")
         self.assertRaises(TypeError, complex, "1", "2")
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -1013,6 +1013,7 @@
             self.assertNotEqual(wr(), None)
         else:
             self.fail("RecursionError not raised")
+        gc_collect()
         self.assertEqual(wr(), None)
 
     def test_errno_ENOTDIR(self):
diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -55,11 +55,12 @@
 SIGNED_CERTFILE2 = data_file("keycert4.pem")
 SIGNING_CA = data_file("pycacert.pem")
 
-SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem")
+REMOTE_HOST = "self-signed.pythontest.net"
+REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem")
 
 EMPTYCERT = data_file("nullcert.pem")
 BADCERT = data_file("badcert.pem")
-WRONGCERT = data_file("XXXnonexisting.pem")
+NONEXISTINGCERT = data_file("XXXnonexisting.pem")
 BADKEY = data_file("badkey.pem")
 NOKIACERT = data_file("nokia.pem")
 NULLBYTECERT = data_file("nullbytecert.pem")
@@ -276,7 +277,7 @@
         self.assertEqual(p['subjectAltName'], san)
 
     def test_DER_to_PEM(self):
-        with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f:
+        with open(CAFILE_CACERT, 'r') as f:
             pem = f.read()
         d1 = ssl.PEM_cert_to_DER_cert(pem)
         p2 = ssl.DER_cert_to_PEM_cert(d1)
@@ -366,17 +367,42 @@
                                     s.connect, (HOST, 8080))
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT)
+                ssl.wrap_socket(sock, certfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=CERTFILE, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(OSError) as cm:
             with socket.socket() as sock:
-                ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT)
+                ssl.wrap_socket(sock,
+                    certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
 
+    def bad_cert_test(self, certfile):
+        """Check that trying to use the given client certificate fails"""
+        certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                   certfile)
+        sock = socket.socket()
+        self.addCleanup(sock.close)
+        with self.assertRaises(ssl.SSLError):
+            ssl.wrap_socket(sock,
+                            certfile=certfile,
+                            ssl_version=ssl.PROTOCOL_TLSv1)
+
+    def test_empty_cert(self):
+        """Wrapping with an empty cert file"""
+        self.bad_cert_test("nullcert.pem")
+
+    def test_malformed_cert(self):
+        """Wrapping with a badly formatted certificate (syntax error)"""
+        self.bad_cert_test("badcert.pem")
+
+    def test_malformed_key(self):
+        """Wrapping with a badly formatted key (syntax error)"""
+        self.bad_cert_test("badkey.pem")
+
     def test_match_hostname(self):
         def ok(cert, hostname):
             ssl.match_hostname(cert, hostname)
@@ -795,7 +821,8 @@
             self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3,
                              ctx.options)
             ctx.options = 0
-            self.assertEqual(0, ctx.options)
+            # Ubuntu has OP_NO_SSLv3 forced on by default
+            self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3)
         else:
             with self.assertRaises(ValueError):
                 ctx.options = 0
@@ -842,7 +869,7 @@
         ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE)
         self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE)
         with self.assertRaises(OSError) as cm:
-            ctx.load_cert_chain(WRONGCERT)
+            ctx.load_cert_chain(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_cert_chain(BADCERT)
@@ -862,7 +889,7 @@
         # Mismatching key and cert
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"):
-            ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY)
+            ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY)
         # Password protected key and cert
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD)
         ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode())
@@ -927,7 +954,7 @@
         self.assertRaises(TypeError, ctx.load_verify_locations)
         self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None)
         with self.assertRaises(OSError) as cm:
-            ctx.load_verify_locations(WRONGCERT)
+            ctx.load_verify_locations(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaisesRegex(ssl.SSLError, "PEM lib"):
             ctx.load_verify_locations(BADCERT)
@@ -1003,7 +1030,7 @@
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
         with self.assertRaises(FileNotFoundError) as cm:
-            ctx.load_dh_params(WRONGCERT)
+            ctx.load_dh_params(NONEXISTINGCERT)
         self.assertEqual(cm.exception.errno, errno.ENOENT)
         with self.assertRaises(ssl.SSLError) as cm:
             ctx.load_dh_params(CERTFILE)
@@ -1080,7 +1107,7 @@
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 0, 'crl': 0, 'x509': 1})
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.cert_store_stats(),
             {'x509_ca': 1, 'crl': 0, 'x509': 2})
 
@@ -1090,8 +1117,8 @@
         # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE
         ctx.load_verify_locations(CERTFILE)
         self.assertEqual(ctx.get_ca_certs(), [])
-        # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert
-        ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+        # but CAFILE_CACERT is a CA cert
+        ctx.load_verify_locations(CAFILE_CACERT)
         self.assertEqual(ctx.get_ca_certs(),
             [{'issuer': ((('organizationName', 'Root CA'),),
                          (('organizationalUnitName', 'http://www.cacert.org'),),
@@ -1107,7 +1134,7 @@
                           (('emailAddress', 'support at cacert.org'),)),
               'version': 3}])
 
-        with open(SVN_PYTHON_ORG_ROOT_CERT) as f:
+        with open(CAFILE_CACERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
         self.assertEqual(ctx.get_ca_certs(True), [der])
@@ -1345,11 +1372,11 @@
 class NetworkedTests(unittest.TestCase):
 
     def test_connect(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_NONE)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
@@ -1358,27 +1385,27 @@
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED)
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                   s.connect, ("svn.python.org", 443))
+                                   s.connect, (REMOTE_HOST, 443))
             s.close()
 
             # this should succeed because we specify the root cert
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
 
     def test_connect_ex(self):
         # Issue #11326: check connect_ex() implementation
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                self.assertEqual(0, s.connect_ex(("svn.python.org", 443)))
+                self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443)))
                 self.assertTrue(s.getpeercert())
             finally:
                 s.close()
@@ -1386,14 +1413,14 @@
     def test_non_blocking_connect_ex(self):
         # Issue #11326: non-blocking connect_ex() should allow handshake
         # to proceed after the socket gets ready.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.setblocking(False)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 # EWOULDBLOCK under Windows, EINPROGRESS elsewhere
                 self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK))
                 # Wait for connect to finish
@@ -1415,58 +1442,62 @@
     def test_timeout_connect_ex(self):
         # Issue #12065: on a timeout, connect_ex() should return the original
         # errno (mimicking the behaviour of non-SSL sockets).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT,
+                                ca_certs=REMOTE_ROOT_CERT,
                                 do_handshake_on_connect=False)
             try:
                 s.settimeout(0.0000001)
-                rc = s.connect_ex(('svn.python.org', 443))
+                rc = s.connect_ex((REMOTE_HOST, 443))
                 if rc == 0:
-                    self.skipTest("svn.python.org responded too quickly")
+                    self.skipTest("REMOTE_HOST responded too quickly")
                 self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK))
             finally:
                 s.close()
 
     def test_connect_ex_error(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = ssl.wrap_socket(socket.socket(socket.AF_INET),
                                 cert_reqs=ssl.CERT_REQUIRED,
-                                ca_certs=SVN_PYTHON_ORG_ROOT_CERT)
+                                ca_certs=REMOTE_ROOT_CERT)
             try:
-                rc = s.connect_ex(("svn.python.org", 444))
+                rc = s.connect_ex((REMOTE_HOST, 444))
                 # Issue #19919: Windows machines or VMs hosted on Windows
                 # machines sometimes return EWOULDBLOCK.
-                self.assertIn(rc, (errno.ECONNREFUSED, errno.EWOULDBLOCK))
+                errors = (
+                    errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT,
+                    errno.EWOULDBLOCK,
+                )
+                self.assertIn(rc, errors)
             finally:
                 s.close()
 
     def test_connect_with_context(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             # Same as test_connect, but with a separately created context
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 self.assertEqual({}, s.getpeercert())
             finally:
                 s.close()
             # Same with a server hostname
             s = ctx.wrap_socket(socket.socket(socket.AF_INET),
-                                server_hostname="svn.python.org")
-            s.connect(("svn.python.org", 443))
+                                server_hostname=REMOTE_HOST)
+            s.connect((REMOTE_HOST, 443))
             s.close()
             # This should fail because we have no verification certs
             ctx.verify_mode = ssl.CERT_REQUIRED
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
             self.assertRaisesRegex(ssl.SSLError, "certificate verify failed",
-                                    s.connect, ("svn.python.org", 443))
+                                    s.connect, (REMOTE_HOST, 443))
             s.close()
             # This should succeed because we specify the root cert
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1479,12 +1510,12 @@
         # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must
         # contain both versions of each certificate (same content, different
         # filename) for this test to be portable across OpenSSL releases.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1495,7 +1526,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=BYTES_CAPATH)
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1503,15 +1534,15 @@
                 s.close()
 
     def test_connect_cadata(self):
-        with open(CAFILE_CACERT) as f:
+        with open(REMOTE_ROOT_CERT) as f:
             pem = f.read()
         der = ssl.PEM_cert_to_DER_cert(pem)
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=pem)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1520,7 +1551,7 @@
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(cadata=der)
             with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
-                s.connect(("svn.python.org", 443))
+                s.connect((REMOTE_HOST, 443))
                 cert = s.getpeercert()
                 self.assertTrue(cert)
 
@@ -1529,9 +1560,9 @@
         # Issue #5238: creating a file-like object with makefile() shouldn't
         # delay closing the underlying "real socket" (here tested with its
         # file descriptor, hence skipping the test under Windows).
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ss = ssl.wrap_socket(socket.socket(socket.AF_INET))
-            ss.connect(("svn.python.org", 443))
+            ss.connect((REMOTE_HOST, 443))
             fd = ss.fileno()
             f = ss.makefile()
             f.close()
@@ -1545,9 +1576,9 @@
             self.assertEqual(e.exception.errno, errno.EBADF)
 
     def test_non_blocking_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             s = socket.socket(socket.AF_INET)
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             s.setblocking(False)
             s = ssl.wrap_socket(s,
                                 cert_reqs=ssl.CERT_NONE,
@@ -1590,12 +1621,12 @@
                 if support.verbose:
                     sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem))
 
-        _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT)
+        _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT)
         if support.IPV6_ENABLED:
             _test_get_server_certificate('ipv6.google.com', 443)
 
     def test_ciphers(self):
-        remote = ("svn.python.org", 443)
+        remote = (REMOTE_HOST, 443)
         with support.transient_internet(remote[0]):
             with ssl.wrap_socket(socket.socket(socket.AF_INET),
                                  cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s:
@@ -1640,13 +1671,13 @@
 
     def test_get_ca_certs_capath(self):
         # capath certs are loaded on request
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
             ctx.load_verify_locations(capath=CAPATH)
             self.assertEqual(ctx.get_ca_certs(), [])
             s = ctx.wrap_socket(socket.socket(socket.AF_INET))
-            s.connect(("svn.python.org", 443))
+            s.connect((REMOTE_HOST, 443))
             try:
                 cert = s.getpeercert()
                 self.assertTrue(cert)
@@ -1657,12 +1688,12 @@
     @needs_sni
     def test_context_setget(self):
         # Check that the context of a connected socket can be replaced.
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             s = socket.socket(socket.AF_INET)
             with ctx1.wrap_socket(s) as ss:
-                ss.connect(("svn.python.org", 443))
+                ss.connect((REMOTE_HOST, 443))
                 self.assertIs(ss.context, ctx1)
                 self.assertIs(ss._sslobj.context, ctx1)
                 ss.context = ctx2
@@ -1683,13 +1714,8 @@
             try:
                 ret = func(*args)
             except ssl.SSLError as e:
-                # Note that we get a spurious -1/SSL_ERROR_SYSCALL for
-                # non-blocking IO. The SSL_shutdown manpage hints at this.
-                # It *should* be safe to just ignore SYS_ERROR_SYSCALL because
-                # with a Memory BIO there's no syscalls (for IO at least).
                 if e.errno not in (ssl.SSL_ERROR_WANT_READ,
-                                   ssl.SSL_ERROR_WANT_WRITE,
-                                   ssl.SSL_ERROR_SYSCALL):
+                                   ssl.SSL_ERROR_WANT_WRITE):
                     raise
                 errno = e.errno
             # Get any data from the outgoing BIO irrespective of any error, and
@@ -1712,36 +1738,46 @@
         return ret
 
     def test_handshake(self):
-        with support.transient_internet("svn.python.org"):
+        # NOTE: this test has been modified, CPython in newer versions
+        # removed the ability to get the shared ciphers of the session, but
+        # they always return the cipher of the ssl context. This test is fully
+        # removed in later versions
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             ctx.verify_mode = ssl.CERT_REQUIRED
-            ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT)
+            ctx.load_verify_locations(REMOTE_ROOT_CERT)
             ctx.check_hostname = True
-            sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org')
+            sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST)
             self.assertIs(sslobj._sslobj.owner, sslobj)
             self.assertIsNone(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertRaises(ValueError, sslobj.getpeercert)
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertIsNone(sslobj.get_channel_binding('tls-unique'))
             self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake)
             self.assertTrue(sslobj.cipher())
-            self.assertIsNone(sslobj.shared_ciphers())
+            #self.assertIsNone(sslobj.shared_ciphers())
             self.assertTrue(sslobj.getpeercert())
             if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES:
                 self.assertTrue(sslobj.get_channel_binding('tls-unique'))
-            self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            try:
+                self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap)
+            except ssl.SSLSyscallError:
+                # self-signed.pythontest.net probably shuts down the TCP
+                # connection without sending a secure shutdown message, and
+                # this is reported as SSL_ERROR_SYSCALL
+                pass
             self.assertRaises(ssl.SSLError, sslobj.write, b'foo')
             sock.close()
 
     def test_read_write_data(self):
-        with support.transient_internet("svn.python.org"):
+        with support.transient_internet(REMOTE_HOST):
             sock = socket.socket(socket.AF_INET)
-            sock.connect(("svn.python.org", 443))
+            sock.connect((REMOTE_HOST, 443))
             incoming = ssl.MemoryBIO()
             outgoing = ssl.MemoryBIO()
             ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -1838,14 +1874,7 @@
                 else:
                     self.sock.close()
 
-            # PyPy change
             def run(self):
-                try:
-                    self._run()
-                finally:
-                    self.close()
-
-            def _run(self):
                 self.running = True
                 if not self.server.starttls_server:
                     if not self.wrap_conn():
@@ -2091,36 +2120,6 @@
             self.active = False
             self.server.close()
 
-    def bad_cert_test(certfile):
-        """
-        Launch a server with CERT_REQUIRED, and check that trying to
-        connect to it with the given client certificate fails.
-        """
-        server = ThreadedEchoServer(CERTFILE,
-                                    certreqs=ssl.CERT_REQUIRED,
-                                    cacerts=CERTFILE, chatty=False,
-                                    connectionchatty=False)
-        with server:
-            try:
-                with socket.socket() as sock:
-                    s = ssl.wrap_socket(sock,
-                                        certfile=certfile,
-                                        ssl_version=ssl.PROTOCOL_TLSv1)
-                    s.connect((HOST, server.port))
-            except ssl.SSLError as x:
-                if support.verbose:
-                    sys.stdout.write("\nSSLError is %s\n" % x.args[1])
-            except OSError as x:
-                if support.verbose:
-                    sys.stdout.write("\nOSError is %s\n" % x.args[1])
-            except OSError as x:
-                if x.errno != errno.ENOENT:
-                    raise
-                if support.verbose:
-                    sys.stdout.write("\OSError is %s\n" % str(x))
-            else:
-                raise AssertionError("Use of invalid cert should have failed!")
-
     def server_params_test(client_context, server_context, indata=b"FOO\n",
                            chatty=True, connectionchatty=False, sni_name=None):
         """
@@ -2361,22 +2360,38 @@
                                                 "check_hostname requires server_hostname"):
                         context.wrap_socket(s)
 
-        def test_empty_cert(self):
-            """Connecting with an empty cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                      "nullcert.pem"))
-        def test_malformed_cert(self):
-            """Connecting with a badly formatted certificate (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badcert.pem"))
-        def test_nonexisting_cert(self):
-            """Connecting with a non-existing cert file"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "wrongcert.pem"))
-        def test_malformed_key(self):
-            """Connecting with a badly formatted key (syntax error)"""
-            bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir,
-                                       "badkey.pem"))
+        def test_wrong_cert(self):
+            """Connecting when the server rejects the client's certificate
+
+            Launch a server with CERT_REQUIRED, and check that trying to
+            connect to it with a wrong client certificate fails.
+            """
+            certfile = os.path.join(os.path.dirname(__file__) or os.curdir,
+                                       "wrongcert.pem")
+            server = ThreadedEchoServer(CERTFILE,
+                                        certreqs=ssl.CERT_REQUIRED,
+                                        cacerts=CERTFILE, chatty=False,
+                                        connectionchatty=False)
+            with server, \
+                    socket.socket() as sock, \
+                    ssl.wrap_socket(sock,
+                                        certfile=certfile,
+                                        ssl_version=ssl.PROTOCOL_TLSv1) as s:
+                try:
+                    # Expect either an SSL error about the server rejecting
+                    # the connection, or a low-level connection reset (which
+                    # sometimes happens on Windows)
+                    s.connect((HOST, server.port))
+                except ssl.SSLError as e:
+                    if support.verbose:
+                        sys.stdout.write("\nSSLError is %r\n" % e)
+                except OSError as e:
+                    if e.errno != errno.ECONNRESET:
+                        raise
+                    if support.verbose:
+                        sys.stdout.write("\nsocket.error is %r\n" % e)
+                else:
+                    self.fail("Use of invalid cert should have failed!")
 
         def test_rude_shutdown(self):
             """A brutal shutdown of an SSL server should raise an OSError
@@ -2782,6 +2797,20 @@
                         # consume data
                         s.read()
 
+                data = b"data"
+
+                # read(-1, buffer) is supported, even though read(-1) is not
+                s.send(data)
+                buffer = bytearray(len(data))
+                self.assertEqual(s.read(-1, buffer), len(data))
+                self.assertEqual(buffer, data)
+
+                # recv/read(0) should return no data
+                s.send(data)
+                self.assertEqual(s.recv(0), b"")
+                self.assertEqual(s.read(0), b"")
+                self.assertEqual(s.read(), data)
+
                 # Make sure sendmsg et al are disallowed to avoid
                 # inadvertent disclosure of data and/or corruption
                 # of the encrypted data stream
@@ -2791,6 +2820,10 @@
                                   s.recvmsg_into, bytearray(100))
 
                 s.write(b"over\n")
+
+                self.assertRaises(ValueError, s.recv, -1)
+                self.assertRaises(ValueError, s.read, -1)
+
                 s.close()
 
         def test_nonblocking_send(self):
@@ -3253,18 +3286,31 @@
             self.assertIn("TypeError", stderr.getvalue())
 
         def test_shared_ciphers(self):
+            # NOTE: This test case has been copied from master (~3.6)
+            # reason is the method SSL_get_ciphers (before that it used
+            # get_ciphers of the session which is not public)
             server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             server_context.load_cert_chain(SIGNED_CERTFILE)
             client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
             client_context.verify_mode = ssl.CERT_REQUIRED
             client_context.load_verify_locations(SIGNING_CA)
-            client_context.set_ciphers("RC4")
-            server_context.set_ciphers("AES:RC4")
+            if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
+                client_context.set_ciphers("AES128:AES256")
+                server_context.set_ciphers("AES256")
+                alg1 = "AES256"
+                alg2 = "AES-256"
+            else:
+                client_context.set_ciphers("AES:3DES")
+                server_context.set_ciphers("3DES")
+                alg1 = "3DES"
+                alg2 = "DES-CBC3"
+
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                self.assertIn("RC4", name.split("-"))
+                if not alg1 in name.split("-") and alg2 not in name:
+                    self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
             context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
@@ -3332,7 +3378,7 @@
             pass
 
     for filename in [
-        CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE,
+        CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE,
         ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY,
         SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA,
         BADCERT, BADKEY, EMPTYCERT]:
diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py
--- a/lib-python/3/test/test_sys.py
+++ b/lib-python/3/test/test_sys.py
@@ -811,7 +811,12 @@
             ref = AtExit()
         """
         rc, stdout, stderr = assert_python_ok('-c', code)
-        self.assertEqual(stdout.rstrip(), b'True')
+        if test.support.check_impl_detail(cpython=True):
+            self.assertEqual(stdout.rstrip(), b'True')
+        else:
+            # the __del__ method may or may not have been called
+            # in other Python implementations
+            self.assertIn(stdout.rstrip(), {b'True', b''})
 
 
 @test.support.cpython_only
diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py
--- a/lib-python/3/test/test_weakref.py
+++ b/lib-python/3/test/test_weakref.py
@@ -1103,7 +1103,7 @@
         # Keep an iterator alive
         it = dct.items()
         try:
-            next(it)
+            print(next(it))
         except StopIteration:
             pass
         del items
@@ -1112,7 +1112,10 @@
         del it
         gc.collect()
         gc.collect()
+        print(list(dct.items()))
         n2 = len(dct)
+        print(len(dct))
+        print(weakref)
         # one item may be kept alive inside the iterator
         self.assertIn(n1, (0, 1))
         self.assertEqual(n2, 0)
diff --git a/lib-python/3/test/wrongcert.pem b/lib-python/3/test/wrongcert.pem
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/wrongcert.pem
@@ -0,0 +1,32 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH
+FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T
+f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB
+AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq
+1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW
+7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg
+SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe
+19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg
+ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666
+lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs
+ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv
+frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk
+2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6
++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK
+24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G
+A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9
+OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt
+U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ
+fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E
+usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+
+43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw
+eSHj5jpC8iZKjCHBn+mAi4cQ514=
+-----END CERTIFICATE-----
diff --git a/lib-python/3/unittest/test/test_assertions.py b/lib-python/3/unittest/test/test_assertions.py
--- a/lib-python/3/unittest/test/test_assertions.py
+++ b/lib-python/3/unittest/test/test_assertions.py
@@ -2,6 +2,7 @@
 import warnings
 import weakref
 import unittest
+from test.support import gc_collect
 from itertools import product
 
 
@@ -124,8 +125,10 @@
                     self.foo()
 
         Foo("test_functional").run()
+        gc_collect()
         self.assertIsNone(wr())
         Foo("test_with").run()
+        gc_collect()
         self.assertIsNone(wr())
 
     def testAssertNotRegex(self):
diff --git a/lib-python/3/unittest/test/test_case.py b/lib-python/3/unittest/test/test_case.py
--- a/lib-python/3/unittest/test/test_case.py
+++ b/lib-python/3/unittest/test/test_case.py
@@ -18,7 +18,7 @@
     TestEquality, TestHashing, LoggingResult, LegacyLoggingResult,
     ResultWithNoStartTestRunStopTestRun
 )
-from test.support import captured_stderr
+from test.support import captured_stderr, gc_collect
 
 
 log_foo = logging.getLogger('foo')
@@ -1737,6 +1737,7 @@
         for method_name in ('test1', 'test2'):
             testcase = TestCase(method_name)
             testcase.run()
+            gc_collect()
             self.assertEqual(MyException.ninstance, 0)
 
 
diff --git a/lib-python/3/weakref.py b/lib-python/3/weakref.py
--- a/lib-python/3/weakref.py
+++ b/lib-python/3/weakref.py
@@ -147,7 +147,14 @@
         del self.data[key]
 
     def __len__(self):
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data.values():
+            result += (wr() is not None)
+        return result
 
     def __contains__(self, key):
         try:
@@ -376,11 +383,18 @@
         return self.data[ref(key)]
 
     def __len__(self):
-        if self._dirty_len and self._pending_removals:
-            # self._pending_removals may still contain keys which were
-            # explicitly removed, we have to scrub them (see issue #21173).
-            self._scrub_removals()
-        return len(self.data) - len(self._pending_removals)
+        # PyPy change: we can't rely on len(self.data) at all, because
+        # the weakref callbacks may be called at an unknown later time.
+#        if self._dirty_len and self._pending_removals:
+#            # self._pending_removals may still contain keys which were
+#            # explicitly removed, we have to scrub them (see issue #21173).
+#            self._scrub_removals()
+#        return len(self.data) - len(self._pending_removals)
+#
+        result = 0
+        for wr in self.data:
+            result += (wr() is not None)
+        return result
 
     def __repr__(self):
         return "<%s at %#x>" % (self.__class__.__name__, id(self))
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Vim
+
+*.swp
+*.swo
diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/LICENSE
@@ -0,0 +1,26 @@
+
+Except when otherwise stated (look for LICENSE files in directories or
+information at the beginning of each file) all software and
+documentation is licensed as follows: 
+
+    The MIT License
+
+    Permission is hereby granted, free of charge, to any person 
+    obtaining a copy of this software and associated documentation 
+    files (the "Software"), to deal in the Software without 
+    restriction, including without limitation the rights to use, 
+    copy, modify, merge, publish, distribute, sublicense, and/or 
+    sell copies of the Software, and to permit persons to whom the 
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included 
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    DEALINGS IN THE SOFTWARE.
+
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -0,0 +1,16 @@
+# PyPy's SSL module
+
+All of the CFFI code is copied from cryptography, wich patches contributed
+back to cryptography. PyPy vendors it's own copy of the cffi backend thus
+it renames the compiled shared object to _pypy_openssl.so (which means
+that cryptography can ship their own cffi backend)
+
+# Tests?
+
+Currently this module is tested using CPython's standard library test suite.
+
+# Install it into PyPy's source tree
+
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:
+
+    $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..180c02ff82d3363f34a334aae22c9876d4c96481
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py
@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "seccertificate",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "secpolicy",
+        "sectransform",
+        "sectrust",
+        "secure_transport",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py
@@ -0,0 +1,85 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding, compiler_type, extra_link_args
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform == "darwin":
+        return _osx_libraries(
+            os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS")
+        )
+    elif platform == "win32":
+        if compiler_type() == "msvc":
+            libs = ["libeay32", "ssleay32"]
+        else:
+            libs = ["ssl", "crypto"]
+        return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+
+
+def _osx_libraries(build_static):
+    # For building statically we don't want to pass the -lssl or -lcrypto flags
+    if build_static == "1":
+        return []
+    else:
+        return ["ssl", "crypto"]
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        # This goes first so we can define some cryptography-wide symbols.
+        "cryptography",
+
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "ocsp",
+        "opensslv",
+        "pem",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy",
+        "pkcs7",
+        "callbacks",
+    ],
+    libraries=_get_openssl_libraries(sys.platform),
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi, compiler_type, extra_link_args
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/padding.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_padding",
+    cdef_source=types,
+    verify_source=functions,
+    extra_link_args=extra_link_args(compiler_type()),
+)
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py
@@ -0,0 +1,5 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py
@@ -0,0 +1,113 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CoreFoundation/CoreFoundation.h>
+"""
+
+TYPES = """
+typedef bool Boolean;
+typedef signed long OSStatus;
+typedef unsigned char UInt8;
+typedef uint32_t UInt32;
+
+typedef const void * CFAllocatorRef;
+const CFAllocatorRef kCFAllocatorDefault;
+typedef ... *CFDataRef;
+typedef signed long long CFIndex;
+typedef ... *CFStringRef;
+typedef ... *CFArrayRef;
+typedef ... *CFMutableArrayRef;
+typedef ... *CFBooleanRef;
+typedef ... *CFErrorRef;
+typedef ... *CFNumberRef;
+typedef ... *CFTypeRef;
+typedef ... *CFDictionaryRef;
+typedef ... *CFMutableDictionaryRef;
+typedef struct {
+    ...;
+} CFDictionaryKeyCallBacks;
+typedef struct {
+    ...;
+} CFDictionaryValueCallBacks;
+typedef struct {
+    ...;
+} CFRange;
+typedef struct {
+    ...;
+} CFArrayCallBacks;
+
+typedef UInt32 CFStringEncoding;
+enum {
+    kCFStringEncodingASCII = 0x0600
+};
+
+enum {
+   kCFNumberSInt8Type = 1,
+   kCFNumberSInt16Type = 2,
+   kCFNumberSInt32Type = 3,
+   kCFNumberSInt64Type = 4,
+   kCFNumberFloat32Type = 5,
+   kCFNumberFloat64Type = 6,
+   kCFNumberCharType = 7,
+   kCFNumberShortType = 8,
+   kCFNumberIntType = 9,
+   kCFNumberLongType = 10,
+   kCFNumberLongLongType = 11,
+   kCFNumberFloatType = 12,
+   kCFNumberDoubleType = 13,
+   kCFNumberCFIndexType = 14,
+   kCFNumberNSIntegerType = 15,
+   kCFNumberCGFloatType = 16,
+   kCFNumberMaxType = 16
+};
+typedef int CFNumberType;
+
+const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks;
+const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks;
+
+const CFArrayCallBacks kCFTypeArrayCallBacks;
+
+const CFBooleanRef kCFBooleanTrue;
+const CFBooleanRef kCFBooleanFalse;
+"""
+
+FUNCTIONS = """
+CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex);
+CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *,
+                                      CFStringEncoding);
+CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **,
+                                   const void **, CFIndex,
+                                   const CFDictionaryKeyCallBacks *,
+                                   const CFDictionaryValueCallBacks *);
+CFMutableDictionaryRef CFDictionaryCreateMutable(
+    CFAllocatorRef,
+    CFIndex,
+    const CFDictionaryKeyCallBacks *,
+    const CFDictionaryValueCallBacks *
+);
+void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *);
+CFIndex CFArrayGetCount(CFArrayRef);
+const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex);
+CFIndex CFDataGetLength(CFDataRef);
+void CFDataGetBytes(CFDataRef, CFRange, UInt8 *);
+CFRange CFRangeMake(CFIndex, CFIndex);
+void CFShow(CFTypeRef);
+Boolean CFBooleanGetValue(CFBooleanRef);
+CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *);
+void CFRelease(CFTypeRef);
+CFTypeRef CFRetain(CFTypeRef);
+
+CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex,
+                                       const CFArrayCallBacks *);
+void CFArrayAppendValue(CFMutableArrayRef, const void *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py
@@ -0,0 +1,99 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonCryptor.h>
+"""
+
+TYPES = """
+enum {
+    kCCAlgorithmAES128 = 0,
+    kCCAlgorithmDES,
+    kCCAlgorithm3DES,
+    kCCAlgorithmCAST,
+    kCCAlgorithmRC4,
+    kCCAlgorithmRC2,
+    kCCAlgorithmBlowfish
+};
+typedef uint32_t CCAlgorithm;
+enum {
+    kCCSuccess = 0,
+    kCCParamError = -4300,
+    kCCBufferTooSmall = -4301,
+    kCCMemoryFailure = -4302,
+    kCCAlignmentError = -4303,
+    kCCDecodeError = -4304,
+    kCCUnimplemented = -4305
+};
+typedef int32_t CCCryptorStatus;
+typedef uint32_t CCOptions;
+enum {
+    kCCEncrypt = 0,
+    kCCDecrypt,
+};
+typedef uint32_t CCOperation;
+typedef ... *CCCryptorRef;
+
+enum {
+    kCCModeOptionCTR_LE = 0x0001,
+    kCCModeOptionCTR_BE = 0x0002
+};
+
+typedef uint32_t CCModeOptions;
+
+enum {
+    kCCModeECB = 1,
+    kCCModeCBC = 2,
+    kCCModeCFB = 3,
+    kCCModeCTR = 4,
+    kCCModeF8 = 5,
+    kCCModeLRW = 6,
+    kCCModeOFB = 7,
+    kCCModeXTS = 8,
+    kCCModeRC4 = 9,
+    kCCModeCFB8 = 10,
+    kCCModeGCM = 11
+};
+typedef uint32_t CCMode;
+enum {
+    ccNoPadding = 0,
+    ccPKCS7Padding = 1,
+};
+typedef uint32_t CCPadding;
+"""
+
+FUNCTIONS = """
+CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm,
+                                        CCPadding, const void *, const void *,
+                                        size_t, const void *, size_t, int,
+                                        CCModeOptions, CCCryptorRef *);
+CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions,
+                                const void *, size_t, const void *,
+                                CCCryptorRef *);
+CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *,
+                                size_t, size_t *);
+CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *);
+CCCryptorStatus CCCryptorRelease(CCCryptorRef);
+
+CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t);
+CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t,
+                                    void *);
+CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *);
+CCCryptorStatus CCCryptorGCMReset(CCCryptorRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+/* Not defined in the public header */
+enum {
+    kCCModeGCM = 11
+};
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py
@@ -0,0 +1,58 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonDigest.h>
+"""
+
+TYPES = """
+typedef uint32_t CC_LONG;
+typedef uint64_t CC_LONG64;
+typedef struct CC_MD5state_st {
+    ...;
+} CC_MD5_CTX;
+typedef struct CC_SHA1state_st {
+    ...;
+} CC_SHA1_CTX;
+typedef struct CC_SHA256state_st {
+    ...;
+} CC_SHA256_CTX;
+typedef struct CC_SHA512state_st {
+    ...;
+} CC_SHA512_CTX;
+"""
+
+FUNCTIONS = """
+int CC_MD5_Init(CC_MD5_CTX *);
+int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG);
+int CC_MD5_Final(unsigned char *, CC_MD5_CTX *);
+
+int CC_SHA1_Init(CC_SHA1_CTX *);
+int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG);
+int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *);
+
+int CC_SHA224_Init(CC_SHA256_CTX *);
+int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA256_Init(CC_SHA256_CTX *);
+int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG);
+int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *);
+
+int CC_SHA384_Init(CC_SHA512_CTX *);
+int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *);
+
+int CC_SHA512_Init(CC_SHA512_CTX *);
+int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG);
+int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py
@@ -0,0 +1,37 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonHMAC.h>
+"""
+
+TYPES = """
+typedef struct {
+    ...;
+} CCHmacContext;
+enum {
+    kCCHmacAlgSHA1,
+    kCCHmacAlgMD5,
+    kCCHmacAlgSHA256,
+    kCCHmacAlgSHA384,
+    kCCHmacAlgSHA512,
+    kCCHmacAlgSHA224
+};
+typedef uint32_t CCHmacAlgorithm;
+"""
+
+FUNCTIONS = """
+void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t);
+void CCHmacUpdate(CCHmacContext *, const void *, size_t);
+void CCHmacFinal(CCHmacContext *, void *);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py
@@ -0,0 +1,39 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonKeyDerivation.h>
+"""
+
+TYPES = """
+enum {
+    kCCPBKDF2 = 2,
+};
+typedef uint32_t CCPBKDFAlgorithm;
+enum {
+    kCCPRFHmacAlgSHA1 = 1,
+    kCCPRFHmacAlgSHA224 = 2,
+    kCCPRFHmacAlgSHA256 = 3,
+    kCCPRFHmacAlgSHA384 = 4,
+    kCCPRFHmacAlgSHA512 = 5,
+};
+typedef uint32_t CCPseudoRandomAlgorithm;
+typedef unsigned int uint;
+"""
+
+FUNCTIONS = """
+int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t,
+                         const uint8_t *, size_t, CCPseudoRandomAlgorithm,
+                         uint, uint8_t *, size_t);
+uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t,
+                      CCPseudoRandomAlgorithm, size_t, uint32_t);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py
@@ -0,0 +1,35 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <CommonCrypto/CommonSymmetricKeywrap.h>
+"""
+
+TYPES = """
+enum {
+    kCCWRAPAES = 1,
+};
+
+typedef uint32_t CCWrappingAlgorithm;
+"""
+
+FUNCTIONS = """
+int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t,
+                        const uint8_t *, size_t, const uint8_t *, size_t,
+                        uint8_t *, size_t *);
+int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *,
+                         const size_t, const uint8_t *, size_t,
+                         const uint8_t *, size_t, uint8_t *, size_t *);
+size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t);
+size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t);
+
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py
@@ -0,0 +1,23 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecCertificate.h>
+"""
+
+TYPES = """
+typedef ... *SecCertificateRef;
+"""
+
+FUNCTIONS = """
+SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef);
+"""
+
+MACROS = """
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py
@@ -0,0 +1,86 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+INCLUDES = """
+#include <Security/SecImportExport.h>
+"""
+
+TYPES = """
+typedef ... *SecAccessRef;
+
+CFStringRef kSecImportExportPassphrase;
+CFStringRef kSecImportExportKeychain;
+CFStringRef kSecImportExportAccess;
+
+typedef uint32_t SecExternalItemType;
+enum {
+    kSecItemTypeUnknown,
+    kSecItemTypePrivateKey,
+    kSecItemTypePublicKey,
+    kSecItemTypeSessionKey,
+    kSecItemTypeCertificate,
+    kSecItemTypeAggregate
+};
+
+
+typedef uint32_t SecExternalFormat;
+enum {
+    kSecFormatUnknown = 0,
+    kSecFormatOpenSSL,
+    kSecFormatSSH,
+    kSecFormatBSAFE,
+    kSecFormatRawKey,
+    kSecFormatWrappedPKCS8,
+    kSecFormatWrappedOpenSSL,
+    kSecFormatWrappedSSH,
+    kSecFormatWrappedLSH,
+    kSecFormatX509Cert,
+    kSecFormatPEMSequence,
+    kSecFormatPKCS7,
+    kSecFormatPKCS12,
+    kSecFormatNetscapeCertSequence,
+    kSecFormatSSHv2
+};
+
+typedef uint32_t SecItemImportExportFlags;
+enum {
+    kSecKeyImportOnlyOne        = 0x00000001,
+    kSecKeySecurePassphrase     = 0x00000002,
+    kSecKeyNoAccessControl      = 0x00000004
+};
+typedef uint32_t SecKeyImportExportFlags;
+
+typedef struct {
+    /* for import and export */
+    uint32_t version;
+    SecKeyImportExportFlags  flags;
+    CFTypeRef                passphrase;
+    CFStringRef              alertTitle;
+    CFStringRef              alertPrompt;
+
+    /* for import only */
+    SecAccessRef             accessRef;
+    CFArrayRef               keyUsage;
+
+    CFArrayRef               keyAttributes;
+} SecItemImportExportKeyParameters;
+"""
+
+FUNCTIONS = """
+OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *,
+                       SecExternalItemType *, SecItemImportExportFlags,

From pypy.commits at gmail.com  Wed Nov 30 09:23:21 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:21 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: special-case 'bytes([single_integer])' or
 'bytes((single_int, ))'
Message-ID: <583ee0d9.c515c20a.cc3bd.d862@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88763:5aa3b36e215f
Date: 2016-11-30 14:39 +0100
http://bitbucket.org/pypy/pypy/changeset/5aa3b36e215f/

Log:	special-case 'bytes([single_integer])' or 'bytes((single_int,))'

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,6 +1,6 @@
 """The builtin bytes implementation"""
 
-from rpython.rlib.jit import we_are_jitted
+from rpython.rlib import jit
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin, newlist_hint,
     resizelist_hint, HASH_ALGORITHM)
@@ -529,10 +529,22 @@
     @unwrap_spec(encoding='str_or_None', errors='str_or_None')
     def descr_new(space, w_stringtype, w_source=None, encoding=None,
                   errors=None):
-        if (w_source and space.is_w(space.type(w_source), space.w_bytes) and
-            space.is_w(w_stringtype, space.w_bytes) and encoding is None
-            and errors is None):
-            return w_source
+        if (w_source and space.is_w(w_stringtype, space.w_bytes)
+                and encoding is None and errors is None):
+            # special-case 'bytes(byte_object)'
+            w_srctype = space.type(w_source)
+            if w_srctype is space.w_bytes:
+                return w_source
+            # special-case 'bytes([single_integer])' or 'bytes((single_int,))'
+            # for JITted performance only, when we clearly see the
+            # length of the list/tuple being constant and equal to 1
+            if w_srctype is space.w_list or w_srctype is space.w_tuple:
+                length = space.len_w(w_source)
+                if jit.isconstant(length) and length == 1:
+                    w_item = space.getitem(w_source, space.wrap(0))
+                    value = getbytevalue(space, w_item)
+                    return W_BytesObject(value)
+        #
         value = newbytesdata_w(space, w_source, encoding, errors)
         w_obj = space.allocate_instance(W_BytesObject, w_stringtype)
         W_BytesObject.__init__(w_obj, value)
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -89,6 +89,13 @@
         assert self.space.listview_bytes(w_bytes) == None
         assert self.space.listview_int(w_bytes) == [97, 98, 99, 100]
 
+    def test_constructor_single_char(self, monkeypatch):
+        from rpython.rlib import jit
+        monkeypatch.setattr(jit, 'isconstant', lambda x: True)
+        space = self.space
+        w_res = space.call_function(space.w_bytes, space.wrap([42]))
+        assert space.str_w(w_res) == '*'
+
 class AppTestBytesObject:
 
     def test_constructor(self):
@@ -97,6 +104,23 @@
         assert bytes(b'abc') == b'abc'
         assert bytes('abc', 'ascii') == b'abc'
         assert bytes(set(b'foo')) in (b'fo', b'of')
+        assert bytes([]) == b''
+        assert bytes([42]) == b'*'
+        assert bytes([0xFC]) == b'\xFC'
+        assert bytes([42, 0xCC]) == b'*\xCC'
+
+    def test_constructor_list_of_objs(self):
+        class X:
+            def __index__(self):
+                return 42
+        class Y:
+            def __int__(self):
+                return 42
+        for obj in [42, X()]:
+            assert bytes([obj]) == b'*'
+            assert bytes([obj, obj, obj]) == b'***'
+        raises(TypeError, bytes, [Y()])
+        raises(TypeError, bytes, [Y(), Y()])
 
     def test_fromhex(self):
         assert bytes.fromhex("abcd") == b'\xab\xcd'

From pypy.commits at gmail.com  Wed Nov 30 09:23:23 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:23 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583ee0db.272cc20a.647e8.cbc5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88764:eb1deae2805b
Date: 2016-11-30 15:05 +0100
http://bitbucket.org/pypy/pypy/changeset/eb1deae2805b/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -21,7 +21,7 @@
             while i < n:
                 buf = struct.pack("<i", i)       # ID: pack
                 x = struct.unpack("<i", buf)[0]  # ID: unpack
-                i += x / i
+                i += x // i
             return i
 
         log = self.run(main, [1000])
@@ -82,7 +82,7 @@
             while i < n:
                 buf = s.pack(-1, i)     # ID: pack
                 x = s.unpack(buf)[1]    # ID: unpack
-                i += x / i
+                i += x // i
             return i
 
         log = self.run(main, [1000])

From pypy.commits at gmail.com  Wed Nov 30 09:23:26 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:26 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: partial test fix
Message-ID: <583ee0de.ce181c0a.42c9d.e33b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88766:8f764936c7cf
Date: 2016-11-30 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/8f764936c7cf/

Log:	partial test fix

diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -38,7 +38,7 @@
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
             ...
-            p76 = call_assembler_r(_, _, _, _, descr=...)
+            p76 = call_assembler_r(..., descr=...)
             ...
         """)
         loop2 = log.loops[0]
@@ -79,6 +79,6 @@
         assert len(guards) < 20
         assert loop.match("""
             ...
-            p76 = call_assembler_r(_, _, _, _, descr=...)
+            p76 = call_assembler_r(..., descr=...)
             ...
         """)

From pypy.commits at gmail.com  Wed Nov 30 09:23:30 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:30 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583ee0e2.272cc20a.647e8.cbd0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88768:0da457b223d3
Date: 2016-11-30 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/0da457b223d3/

Log:	merge heads

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -60,6 +60,8 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
+    so_list = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
+    so_ext = (so_list or ['.so'])[0]
     g = {}
     g['CC'] = "gcc -pthread"
     g['CXX'] = "g++ -pthread"
@@ -67,7 +69,7 @@
     g['CFLAGS'] = "-DNDEBUG -O2"
     g['CCSHARED'] = "-fPIC"
     g['LDSHARED'] = "gcc -pthread -shared"
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['SO'] = so_ext
     g['SHLIB_SUFFIX'] = g['SO']
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"

From pypy.commits at gmail.com  Wed Nov 30 09:23:25 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:25 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583ee0dd.e644c20a.b8aeb.ba57@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88765:a524136be13a
Date: 2016-11-30 14:31 +0000
http://bitbucket.org/pypy/pypy/changeset/a524136be13a/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -22,8 +22,22 @@
         assert log.result == 255
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-            #...
+            # nothing left like allocating a list object or doing any
+            # residual call
+            i49 = int_lt(i38, i26)
+            guard_true(i49, descr=...)
+            guard_not_invalidated(descr=...)
+            i51 = int_lt(i38, 256)
+            guard_true(i51, descr=...)
+            i53 = int_add(i38, 1)
             --TICK--
+            i58 = strlen(p46)
+            i60 = int_add(i58, 1)
+            p61 = newstr(i60)
+            copystrcontent(p46, p61, 0, 0, i58)
+            strsetitem(p61, i58, i38)
+            p62 = newstr(1)
+            strsetitem(p62, 0, i38)
             jump(..., descr=...)
         """)
 

From pypy.commits at gmail.com  Wed Nov 30 09:23:28 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:23:28 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583ee0e0.ca57c20a.49dfd.c3e0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88767:d444146a4f7d
Date: 2016-11-30 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/d444146a4f7d/

Log:	merge heads

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -22,8 +22,22 @@
         assert log.result == 255
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-            #...
+            # nothing left like allocating a list object or doing any
+            # residual call
+            i49 = int_lt(i38, i26)
+            guard_true(i49, descr=...)
+            guard_not_invalidated(descr=...)
+            i51 = int_lt(i38, 256)
+            guard_true(i51, descr=...)
+            i53 = int_add(i38, 1)
             --TICK--
+            i58 = strlen(p46)
+            i60 = int_add(i58, 1)
+            p61 = newstr(i60)
+            copystrcontent(p46, p61, 0, 0, i58)
+            strsetitem(p61, i58, i38)
+            p62 = newstr(1)
+            strsetitem(p62, 0, i38)
             jump(..., descr=...)
         """)
 

From pypy.commits at gmail.com  Wed Nov 30 09:46:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 06:46:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: in-progress
Message-ID: <583ee62d.12921c0a.937c6.eea8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88769:6c76d736e86b
Date: 2016-11-30 14:55 +0000
http://bitbucket.org/pypy/pypy/changeset/6c76d736e86b/

Log:	in-progress

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -55,36 +55,52 @@
         assert log.result == 300
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-            i14 = int_lt(i6, i9)
-            guard_true(i14, descr=...)
+            i88 = int_lt(i83, i36)
+            guard_true(i88, descr=...)
+            p90 = getfield_gc_r(ConstPtr(ptr89), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
-            i16 = int_eq(i6, %d)
-            i19 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i6, i10, descr=<Calli . ii EF=0 OS=14>)
-            i21 = int_lt(i19, 0)
-            guard_false(i21, descr=...)
-            i22 = int_ge(i19, i10)
-            guard_false(i22, descr=...)
-            i23 = strgetitem(p11, i19)
-            i24 = int_ge(i19, i12)
-            guard_false(i24, descr=...)
-            i25 = unicodegetitem(p13, i19)
-            p27 = newstr(1)
-            strsetitem(p27, 0, i23)
-            p30 = call_r(ConstClass(ll_str2unicode__rpy_stringPtr), p27, descr=...)
+            i92 = int_eq(i83, %d)
+            i94 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i83, i46, descr=<Calli . ii EF=0 OS=14>)
+            i96 = int_lt(i94, 0)
+            guard_false(i96, descr=...)
+            i97 = int_ge(i94, i53)
+            guard_false(i97, descr=...)
+            i98 = strgetitem(p52, i94)
+            p100 = getfield_gc_r(ConstPtr(ptr99), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+            p101 = force_token()
+            p103 = newstr(1)
+            strsetitem(p103, 0, i98)
+            p104 = new(descr=<SizeDescr ..?>)
+            p106 = newunicode(1)
+            setfield_gc(p0, p101, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .>)
+            setfield_gc(p104, p106, descr=<FieldP unicodebuilder.current_buf ..?>)
+            setfield_gc(p104, 0, descr=<FieldS unicodebuilder.current_pos ..?>)
+            setfield_gc(p104, 1, descr=<FieldS unicodebuilder.current_end ..?>)
+            setfield_gc(p104, 1, descr=<FieldS unicodebuilder.total_size 32>)
+            i113 = call_may_force_i(ConstClass(str_decode_utf_8_impl), p103, 1, ConstPtr(null), 1, 0, 0, p104, descr=<Calli . ririiir EF=7>)
+            guard_not_forced(descr=...)
             guard_no_exception(descr=...)
-            i32 = call_i(ConstClass(_ll_2_str_eq_checknull_char__rpy_unicodePtr_UniChar), p30, i25, descr=...)
-            guard_true(i32, descr=...)
-            i34 = int_add(i6, 1)
+            p116 = call_r(ConstClass(ll_build_trampoline__v1351___simple_call__function_), p104, descr=<Callr . r EF=5>)
+            guard_no_exception(descr=...)
+            guard_nonnull(p116, descr=...)
+            p118 = getfield_gc_r(ConstPtr(ptr117), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+            guard_not_invalidated(descr=...)
+            i119 = int_ge(i94, i46)
+            guard_false(i119, descr=...)
+            i120 = unicodegetitem(p45, i94)
+            i122 = call_i(ConstClass(_ll_2_str_eq_nonnull_char__rpy_unicodePtr_UniChar), p116, i120, descr=<Calli . ri EF=0 OS=49>)
+            guard_true(i122, descr=...)
+            i124 = int_add(i83, 1)
             --TICK--
             jump(..., descr=...)
         """ % (-sys.maxint-1,))
 
     def test_long(self):
         def main(n):
-            digits = '0123456789'
             i = 1
             while i < n:
-                i += int(long(string.digits[i % len(string.digits)], 16))
+                digits = '0123456789'
+                i += int(digits[i % len(digits)], 16)
             return i
 
         log = self.run(main, [1100], import_site=True)
@@ -97,7 +113,9 @@
         assert loop.match("""
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
+            p70 = getfield_gc_r(ConstPtr(ptr69), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
+            p72 = getfield_gc_r(ConstPtr(ptr71), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             i13 = int_eq(i6, %d)         # value provided below
 
             # "mod 10" block:

From pypy.commits at gmail.com  Wed Nov 30 10:18:08 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:18:08 -0800 (PST)
Subject: [pypy-commit] pypy default: possibly fix the test---the comparison
 might have been removed by the
Message-ID: <583eedb0.6737c20a.58d18.f310@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r88770:226ca1a1aeee
Date: 2016-11-30 16:00 +0100
http://bitbucket.org/pypy/pypy/changeset/226ca1a1aeee/

Log:	possibly fix the test---the comparison might have been removed by
	the AST compiler

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -387,7 +387,8 @@
     def test_long_comparison(self):
         def main(n):
             while n:
-                12345L > 123L  # ID: long_op
+                x = 12345L
+                x > 123L  # ID: long_op
                 n -= 1
 
         log = self.run(main, [300])

From pypy.commits at gmail.com  Wed Nov 30 10:18:10 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:18:10 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Expand the tests for
 types.SimpleNamespace (passing)
Message-ID: <583eedb2.c515c20a.cc3bd.ee9b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88771:93bb01c14908
Date: 2016-11-30 16:16 +0100
http://bitbucket.org/pypy/pypy/changeset/93bb01c14908/

Log:	Expand the tests for types.SimpleNamespace (passing)

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -589,6 +589,28 @@
         #
         raises(AttributeError, "del ns.spam")
         del ns.y
+        #
+        assert ns == SimpleNamespace(z=4, x=1, w=3)
+        assert (ns != SimpleNamespace(z=4, x=1, w=3)) is False
+        assert (ns == SimpleNamespace(z=4, x=2, w=3)) is False
+        assert ns != SimpleNamespace(z=4, x=2, w=3)
+        #
+        class Foo(SimpleNamespace):
+            pass
+        assert ns == Foo(z=4, x=1, w=3)
+        assert (ns != Foo(z=4, x=1, w=3)) is False
+        assert (ns == Foo(z=4, x=2, w=3)) is False
+        assert ns != Foo(z=4, x=2, w=3)
+        #
+        class Other:
+            def __init__(self, x, z, w):
+                self.x = x
+                self.z = z
+                self.w = w
+        assert (ns == Other(z=4, x=1, w=3)) is False
+        assert ns != Other(z=4, x=1, w=3)
+        assert (Foo(z=4, x=1, w=3) == Other(z=4, x=1, w=3)) is False
+        assert Foo(z=4, x=1, w=3) != Other(z=4, x=1, w=3)
 
     def test_pickle_simplenamespace(self):
         import pickle, sys

From pypy.commits at gmail.com  Wed Nov 30 10:24:05 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:24:05 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <583eef15.c8111c0a.6dd96.04ae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88772:804de614b905
Date: 2016-11-30 15:11 +0000
http://bitbucket.org/pypy/pypy/changeset/804de614b905/

Log:	fix tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -247,6 +247,13 @@
         """)
 
     def test_dont_trace_every_iteration(self):
+        def reference(a, b):
+            i = sa = 0
+            while i < 300:
+                sa += a % b
+                i += 1
+            return sa
+        #
         def main(a, b):
             i = sa = 0
             while i < 300:
@@ -258,9 +265,12 @@
                 i += 1
             return sa
         #
+        log_ref = self.run(reference, [10, 20])
+        assert log_ref.result == 300 * (10 % 20)
+        #
         log = self.run(main, [10, 20])
         assert log.result == 300 * (10 % 20)
-        assert log.jit_summary.tracing_no == 1
+        assert log.jit_summary.tracing_no == log_ref.jit_summary.tracing_no
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
             i11 = int_lt(i7, 300)
@@ -274,7 +284,7 @@
         #
         log = self.run(main, [-10, -20])
         assert log.result == 300 * (-10 % -20)
-        assert log.jit_summary.tracing_no == 1
+        assert log.jit_summary.tracing_no == log_ref.jit_summary.tracing_no
 
     def test_overflow_checking(self):
         """
@@ -297,6 +307,7 @@
         self.run_and_check(main, [])
 
     def test_global(self):
+        # check that the global read is removed even from the entry bridge
         log = self.run("""
         i = 0
         globalinc = 1
@@ -308,7 +319,10 @@
         """, [1000])
 
         loop, = log.loops_by_id("globalread", is_entry_bridge=True)
-        assert len(loop.ops_by_id("globalread")) == 0
+        assert loop.match_by_id("globalread", """
+            # only a dead read
+            p26 = getfield_gc_r(ConstPtr(ptr25), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+        """)
 
     def test_eval(self):
         def main():
@@ -349,7 +363,8 @@
     def test_long_comparison(self):
         def main(n):
             while n:
-                12345L > 123L  # ID: long_op
+                x = 12345678901234567890123456
+                x > 1231231231231231231231231  # ID: long_op
                 n -= 1
 
         log = self.run(main, [300])

From pypy.commits at gmail.com  Wed Nov 30 10:24:07 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:24:07 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test, slightly improve code generated
Message-ID: <583eef17.61adc20a.22fed.f491@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88773:4ed461a08c1e
Date: 2016-11-30 15:34 +0000
http://bitbucket.org/pypy/pypy/changeset/4ed461a08c1e/

Log:	fix test, slightly improve code generated

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -95,7 +95,7 @@
             jump(..., descr=...)
         """ % (-sys.maxint-1,))
 
-    def test_long(self):
+    def test_int_base_16(self):
         def main(n):
             i = 1
             while i < n:
@@ -127,17 +127,20 @@
             i87 = int_mul(i85, 10)
             i19 = int_sub(i6, i87)
 
-            i23 = strgetitem(p10, i19)
-            p25 = newstr(1)
-            strsetitem(p25, 0, i23)
-            p93 = call_r(ConstClass(fromstr), p25, 16, descr=<Callr . ri EF=4>)
+            i23 = unicodegetitem(ConstPtr(ptr92), i19)
+            p25 = newunicode(1)
+            unicodesetitem(p25, 0, i23)
+            p97 = call_r(ConstClass(_rpy_unicode_to_decimal_w), p25, descr=<Callr . r EF=5>)
             guard_no_exception(descr=...)
-            i95 = getfield_gc_i(p93, descr=<FieldS rpython.rlib.rbigint.rbigint.inst_size .*>)
-            i96 = int_gt(i95, #)
-            guard_false(i96, descr=...)
-            i94 = call_i(ConstClass(rbigint._toint_helper), p93, descr=<Calli . r EF=4>)
+            i98 = unicodelen(p97)
+            p99 = force_token()
+            setfield_gc(p0, p99, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .>)
+            p104 = call_may_force_r(ConstClass(unicode_encode_utf_8_impl), p97, i98, ConstPtr(ptr101), 1, 1, descr=<Callr . ririi EF=7>)
+            guard_not_forced(descr=...)
             guard_no_exception(descr=...)
-            i95 = int_add_ovf(i6, i94)
+            i107 = call_i(ConstClass(string_to_int), p104, 16, descr=<Calli . ri EF=4>)
+            guard_no_exception(descr=...)
+            i95 = int_add_ovf(i6, i107)
             guard_no_overflow(descr=...)
             --TICK--
             jump(..., descr=...)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1257,7 +1257,11 @@
 def unicode_to_decimal_w(space, w_unistr, allow_surrogates=False):
     if not isinstance(w_unistr, W_UnicodeObject):
         raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
-    unistr = w_unistr._value
+    value = _rpy_unicode_to_decimal_w(space, w_unistr._value)
+    return unicodehelper.encode_utf8(space, value,
+                                     allow_surrogates=allow_surrogates)
+
+def _rpy_unicode_to_decimal_w(space, unistr):
     result = [u'\0'] * len(unistr)
     for i in xrange(len(unistr)):
         uchr = ord(unistr[i])
@@ -1270,8 +1274,7 @@
             except KeyError:
                 pass
         result[i] = unichr(uchr)
-    return unicodehelper.encode_utf8(space, u''.join(result),
-                                     allow_surrogates=allow_surrogates)
+    return u''.join(result)
 
 @jit.elidable
 def g_encode_utf8(value):

From pypy.commits at gmail.com  Wed Nov 30 10:24:09 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:24:09 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <583eef19.e626c20a.f98d5.009b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88774:278d4c3b3ff0
Date: 2016-11-30 15:34 +0000
http://bitbucket.org/pypy/pypy/changeset/278d4c3b3ff0/

Log:	merge heads

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -589,6 +589,28 @@
         #
         raises(AttributeError, "del ns.spam")
         del ns.y
+        #
+        assert ns == SimpleNamespace(z=4, x=1, w=3)
+        assert (ns != SimpleNamespace(z=4, x=1, w=3)) is False
+        assert (ns == SimpleNamespace(z=4, x=2, w=3)) is False
+        assert ns != SimpleNamespace(z=4, x=2, w=3)
+        #
+        class Foo(SimpleNamespace):
+            pass
+        assert ns == Foo(z=4, x=1, w=3)
+        assert (ns != Foo(z=4, x=1, w=3)) is False
+        assert (ns == Foo(z=4, x=2, w=3)) is False
+        assert ns != Foo(z=4, x=2, w=3)
+        #
+        class Other:
+            def __init__(self, x, z, w):
+                self.x = x
+                self.z = z
+                self.w = w
+        assert (ns == Other(z=4, x=1, w=3)) is False
+        assert ns != Other(z=4, x=1, w=3)
+        assert (Foo(z=4, x=1, w=3) == Other(z=4, x=1, w=3)) is False
+        assert Foo(z=4, x=1, w=3) != Other(z=4, x=1, w=3)
 
     def test_pickle_simplenamespace(self):
         import pickle, sys

From pypy.commits at gmail.com  Wed Nov 30 10:27:01 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 30 Nov 2016 07:27:01 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: start replacing space.str_w with
 either space.bytes_w or space.text_w
Message-ID: <583eefc5.12921c0a.937c6.ff83@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88775:ffe62ff416aa
Date: 2016-11-30 16:26 +0100
http://bitbucket.org/pypy/pypy/changeset/ffe62ff416aa/

Log:	start replacing space.str_w with either space.bytes_w or
	space.text_w

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -71,7 +71,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
             try:
@@ -79,7 +79,7 @@
             except OperationError as e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
 
@@ -130,7 +130,7 @@
             if verbose:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
-                debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+                debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
             if must_leave:
@@ -184,7 +184,7 @@
         except OperationError as e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
-            debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space))))
+            debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -77,7 +77,8 @@
 
     def getname(self, space):
         try:
-            return space.str_w(space.getattr(self, space.newtext('__name__')))
+            # YYY should be text_w?
+            return space.bytes_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return '?'
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -64,7 +64,7 @@
             if self.__class__ is not OperationError and s is None:
                 s = self._compute_value(space)
             try:
-                s = space.str_w(s)
+                s = space.text_w(s)
             except Exception:
                 pass
         return '[%s: %s]' % (self.w_type, s)
@@ -77,16 +77,16 @@
             exc_typename = str(self.w_type)
             exc_value = str(w_value)
         else:
-            exc_typename = space.str_w(
+            exc_typename = space.text_w(
                     space.getattr(self.w_type, space.newtext('__name__')))
             if space.is_w(w_value, space.w_None):
                 exc_value = ""
             else:
                 try:
                     if use_repr:
-                        exc_value = space.str_w(space.repr(w_value))
+                        exc_value = space.text_w(space.repr(w_value))
                     else:
-                        exc_value = space.str_w(space.str(w_value))
+                        exc_value = space.text_w(space.str(w_value))
                 except OperationError:
                     # oups, cannot __str__ the exception object
                     exc_value = ("<exception %s() failed>" %
@@ -238,7 +238,7 @@
             objrepr = ''
         else:
             try:
-                objrepr = space.str_w(space.repr(w_object))
+                objrepr = space.text_w(space.repr(w_object))
             except OperationError:
                 objrepr = "<object repr() failed>"
         #
@@ -364,7 +364,7 @@
                     lst[i + i] = self.xstrings[i]
                     value = getattr(self, attr)
                     if fmt == 'R':
-                        result = space.str_w(space.repr(value))
+                        result = space.text_w(space.repr(value))
                     elif fmt == 'T':
                         result = space.type(value).name
                     elif fmt == 'N':
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -213,7 +213,7 @@
         if not space.isinstance_w(w_globals, space.w_dict):
             raise oefmt(space.w_TypeError, "expected dict")
         if not space.is_none(w_name):
-            name = space.str_w(w_name)
+            name = space.text_w(w_name)
         else:
             name = None
         if not space.is_none(w_argdefs):
@@ -377,8 +377,8 @@
         return space.newtext(self.name)
 
     def fset_func_name(self, space, w_name):
-        if space.isinstance_w(w_name, space.w_str):
-            self.name = space.str_w(w_name)
+        if space.isinstance_w(w_name, space.w_text):
+            self.name = space.text_w(w_name)
         else:
             raise oefmt(space.w_TypeError,
                         "__name__ must be set to a string object")
@@ -529,13 +529,13 @@
             s = "<unbound method %s.%s>" % (typename, name)
             return space.newtext(s)
         else:
-            objrepr = space.str_w(space.repr(self.w_instance))
+            objrepr = space.text_w(space.repr(self.w_instance))
             s = '<bound method %s.%s of %s>' % (typename, name, objrepr)
             return space.newtext(s)
 
     def descr_method_getattribute(self, w_attr):
         space = self.space
-        if space.str_w(w_attr) != '__doc__':
+        if space.text_w(w_attr) != '__doc__':
             try:
                 return space.call_method(space.w_object, '__getattribute__',
                                          self, w_attr)
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -122,14 +122,14 @@
     def descr_module__repr__(self, space):
         from pypy.interpreter.mixedmodule import MixedModule
         if self.w_name is not None:
-            name = space.str_w(space.repr(self.w_name))
+            name = space.text_w(space.repr(self.w_name))
         else:
             name = "'?'"
         if isinstance(self, MixedModule):
             return space.newtext("<module %s (built-in)>" % name)
         try:
             w___file__ = space.getattr(self, space.newtext('__file__'))
-            __file__ = space.str_w(space.repr(w___file__))
+            __file__ = space.text_w(space.repr(w___file__))
         except OperationError:
             __file__ = '?'
         return space.newtext("<module %s from %s>" % (name, __file__))
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -848,8 +848,8 @@
             # catch KeyErrors and turn them into NameErrors
             if not e.match(self.space, self.space.w_KeyError):
                 raise
-            raise oefmt(self.space.w_NameError, "name '%s' is not defined",
-                        self.space.str_w(w_varname))
+            raise oefmt(self.space.w_NameError, "name %R is not defined",
+                        w_varname)
 
     def UNPACK_SEQUENCE(self, itemcount, next_instr):
         w_iterable = self.popvalue()
@@ -996,7 +996,6 @@
     def IMPORT_NAME(self, nameindex, next_instr):
         space = self.space
         w_modulename = self.getname_w(nameindex)
-        modulename = self.space.str_w(w_modulename)
         w_fromlist = self.popvalue()
 
         w_flag = self.popvalue()
@@ -1018,7 +1017,6 @@
             w_locals = d.w_locals
         if w_locals is None:            # CPython does this
             w_locals = space.w_None
-        w_modulename = space.newtext(modulename)
         w_globals = self.get_w_globals()
         if w_flag is None:
             w_obj = space.call_function(w_import, w_modulename, w_globals,
@@ -1044,7 +1042,7 @@
             if not e.match(self.space, self.space.w_AttributeError):
                 raise
             raise oefmt(self.space.w_ImportError,
-                        "cannot import name '%s'", self.space.str_w(w_name))
+                        "cannot import name %R", w_name)
         self.pushvalue(w_obj)
 
     def YIELD_VALUE(self, oparg, next_instr):
@@ -1177,7 +1175,7 @@
                     break
                 w_value = self.popvalue()
                 w_key = self.popvalue()
-                key = self.space.str_w(w_key)
+                key = self.space.text_w(w_key)
                 keywords[n_keywords] = key
                 keywords_w[n_keywords] = w_value
         else:
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -41,7 +41,7 @@
             return space.call_function(w_selftype, self.w_starttype, w_obj)
 
     def getattribute(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         # only use a special logic for bound super objects and not for
         # getting the __class__ of the super object itself.
         if self.w_objtype is not None and name != '__class__':
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -42,7 +42,7 @@
     _immutable_fields_ = ['bases_w?[*]', 'w_dict?']
 
     def __init__(self, space, w_name, bases, w_dict):
-        self.name = space.str_w(w_name)
+        self.name = space.text_w(w_name)
         make_sure_not_resized(bases)
         self.bases_w = bases
         self.w_dict = w_dict
@@ -66,7 +66,7 @@
     def setname(self, space, w_newname):
         if not space.isinstance_w(w_newname, space.w_str):
             raise oefmt(space.w_TypeError, "__name__ must be a string object")
-        self.name = space.str_w(w_newname)
+        self.name = space.text_w(w_newname)
 
     def setbases(self, space, w_bases):
         if not space.isinstance_w(w_bases, space.w_tuple):
@@ -123,7 +123,7 @@
         return space.call_function(w_descr_get, w_value, space.w_None, self)
 
     def descr_setattr(self, space, w_attr, w_value):
-        name = space.str_w(w_attr)
+        name = space.text_w(w_attr)
         if name and name[0] == "_":
             if name == "__dict__":
                 self.setdict(space, w_value)
@@ -142,7 +142,7 @@
         space.setitem(self.w_dict, w_attr, w_value)
 
     def descr_delattr(self, space, w_attr):
-        name = space.str_w(w_attr)
+        name = space.text_w(w_attr)
         if name in ("__dict__", "__name__", "__bases__"):
             raise oefmt(space.w_TypeError,
                         "cannot delete attribute '%s'", name)
@@ -173,7 +173,7 @@
                 raise
             return "?"
         if space.isinstance_w(w_mod, space.w_str):
-            return space.str_w(w_mod)
+            return space.text_w(w_mod)
         return "?"
 
     def __repr__(self):
@@ -361,7 +361,7 @@
         return self.getattr(space, name)
 
     def descr_setattr(self, space, w_name, w_value):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_meth = self.getattr_from_class(space, '__setattr__')
         if name and name[0] == "_":
             if name == '__dict__':
@@ -382,7 +382,7 @@
             self.setdictvalue(space, name, w_value)
 
     def descr_delattr(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         if name and name[0] == "_":
             if name == '__dict__':
                 # use setdict to raise the error
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -45,7 +45,7 @@
     # Note that if w_name is already an exact string it must be returned
     # unmodified (and not e.g. unwrapped-rewrapped).
     if not space.is_w(space.type(w_name), space.w_str):
-        name = space.str_w(w_name)    # typecheck
+        name = space.text_w(w_name)    # typecheck
         w_name = space.newtext(name)     # rewrap as a real string
     return w_name
 
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -309,7 +309,7 @@
     if w_encoding is None:
         encoding = space.sys.defaultencoding
     else:
-        encoding = space.str_w(w_encoding)
+        encoding = space.text_w(w_encoding)
     w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
     w_res = space.call_function(w_encoder, w_obj, space.newtext(errors))
     return space.getitem(w_res, space.newint(0))
@@ -338,7 +338,7 @@
     if w_encoding is None:
         encoding = space.sys.defaultencoding
     else:
-        encoding = space.str_w(w_encoding)
+        encoding = space.text_w(w_encoding)
     w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
     if space.is_true(w_decoder):
         w_res = space.call_function(w_decoder, w_obj, space.newtext(errors))
diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py
--- a/pypy/module/_csv/interp_reader.py
+++ b/pypy/module/_csv/interp_reader.py
@@ -79,7 +79,7 @@
                             break
                 raise
             self.line_num += 1
-            line = space.str_w(w_line)
+            line = space.text_w(w_line)
             for c in line:
                 if c == '\0':
                     raise self.error("line contains NULL byte")
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -51,7 +51,7 @@
     if space.is_none(w_locale):
         locale = None
     else:
-        locale = space.str_w(w_locale)
+        locale = space.text_w(w_locale)
     try:
         result = rlocale.setlocale(category, locale)
     except rlocale.LocaleError as e:
@@ -121,10 +121,10 @@
 def strcoll(space, w_s1, w_s2):
     "string,string -> int. Compares two strings according to the locale."
 
-    if (space.isinstance_w(w_s1, space.w_str) and
-        space.isinstance_w(w_s2, space.w_str)):
+    if (space.isinstance_w(w_s1, space.w_bytes) and
+        space.isinstance_w(w_s2, space.w_bytes)):
 
-        s1, s2 = space.str_w(w_s1), space.str_w(w_s2)
+        s1, s2 = space.bytes_w(w_s1), space.bytes_w(w_s2)
         s1_c = rffi.str2charp(s1)
         s2_c = rffi.str2charp(s2)
         try:
@@ -221,7 +221,7 @@
             finally:
                 rffi.free_charp(msg_c)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             msg_c = rffi.str2charp(msg)
             try:
@@ -256,7 +256,7 @@
             finally:
                 rffi.free_charp(msg_c)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             msg_c = rffi.str2charp(msg)
             try:
@@ -284,7 +284,7 @@
             result = _textdomain(domain)
             result = rffi.charp2str(result)
         else:
-            domain = space.str_w(w_domain)
+            domain = space.text_w(w_domain)
             domain_c = rffi.str2charp(domain)
             try:
                 result = _textdomain(domain_c)
@@ -314,7 +314,7 @@
             finally:
                 rffi.free_charp(domain_c)
         else:
-            dir = space.str_w(w_dir)
+            dir = space.text_w(w_dir)
             domain_c = rffi.str2charp(domain)
             dir_c = rffi.str2charp(dir)
             try:
@@ -345,7 +345,7 @@
                 finally:
                     rffi.free_charp(domain_c)
             else:
-                codeset = space.str_w(w_codeset)
+                codeset = space.text_w(w_codeset)
                 domain_c = rffi.str2charp(domain)
                 codeset_c = rffi.str2charp(codeset)
                 try:
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -49,11 +49,11 @@
         return self.w_calls
 
     def repr(self, space):
-        frame_repr = space.str_w(space.repr(self.frame))
+        frame_repr = space.text_w(space.repr(self.frame))
         if not self.w_calls:
             calls_repr = "None"
         else:
-            calls_repr = space.str_w(space.repr(self.w_calls))
+            calls_repr = space.text_w(space.repr(self.w_calls))
         return space.newtext('("%s", %d, %d, %f, %f, %s)' % (
             frame_repr, self.callcount, self.reccallcount,
             self.tt, self.it, calls_repr))
@@ -85,7 +85,7 @@
         self.tt = tt
 
     def repr(self, space):
-        frame_repr = space.str_w(space.repr(self.frame))
+        frame_repr = space.text_w(space.repr(self.frame))
         return space.newtext('("%s", %d, %d, %f, %f)' % (
             frame_repr, self.callcount, self.reccallcount, self.tt, self.it))
 
@@ -218,7 +218,7 @@
 def create_spec_for_function(space, w_func):
     assert isinstance(w_func, Function)
     if w_func.w_module is not None:
-        module = space.str_w(w_func.w_module)
+        module = space.text_w(w_func.w_module)
         if module != '__builtin__':
             return '<%s.%s>' % (module, w_func.name)
     return '<%s>' % w_func.name
@@ -233,7 +233,7 @@
     # This class should not be seen at app-level, but is useful to
     # contain a (w_func, w_type) pair returned by prepare_spec().
     # Turning this pair into a string cannot be done eagerly in
-    # an @elidable function because of space.str_w(), but it can
+    # an @elidable function because of space.text_w(), but it can
     # be done lazily when we really want it.
 
     _immutable_fields_ = ['w_func', 'w_type']
diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py
--- a/pypy/module/_minimal_curses/interp_curses.py
+++ b/pypy/module/_minimal_curses/interp_curses.py
@@ -51,7 +51,7 @@
         if space.is_none(w_termname):
             _curses_setupterm_null(fd)
         else:
-            _curses_setupterm(space.str_w(w_termname), fd)
+            _curses_setupterm(space.text_w(w_termname), fd)
     except curses_error as e:
         raise convert_error(space, e)
 
diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py
--- a/pypy/module/_multibytecodec/interp_incremental.py
+++ b/pypy/module/_multibytecodec/interp_incremental.py
@@ -33,7 +33,7 @@
         return space.newtext(self.errors)
 
     def fset_errors(self, space, w_errors):
-        self.errors = space.str_w(w_errors)
+        self.errors = space.text_w(w_errors)
 
 
 class MultibyteIncrementalDecoder(MultibyteIncrementalBase):
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -156,7 +156,7 @@
         w_pickled = space.call_method(
             w_picklemodule, "dumps", w_obj, w_protocol)
 
-        buf = space.str_w(w_pickled)
+        buf = space.bytes_w(w_pickled)
         self.do_send_string(space, buf, 0, len(buf))
 
     def recv(self, space):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1432,11 +1432,11 @@
         if space.is_none(w_certfile):
             certfile = None
         else:
-            certfile = space.str_w(w_certfile)
+            certfile = space.text_w(w_certfile)
         if space.is_none(w_keyfile):
             keyfile = certfile
         else:
-            keyfile = space.str_w(w_keyfile)
+            keyfile = space.text_w(w_keyfile)
         pw_info = PasswordInfo()
         pw_info.space = space
         index = -1
@@ -1529,11 +1529,11 @@
         if space.is_none(w_cafile):
             cafile = None
         else:
-            cafile = space.str_w(w_cafile)
+            cafile = space.text_w(w_cafile)
         if space.is_none(w_capath):
             capath = None
         else:
-            capath = space.str_w(w_capath)
+            capath = space.text_w(w_capath)
         if space.is_none(w_cadata):
             cadata = None
             ca_file_type = -1
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -101,9 +101,9 @@
     # setup filename
     try:
         w_filename = space.getitem(w_globals, space.newtext("__file__"))
-        filename = space.str_w(w_filename)
+        filename = space.text_w(w_filename)
     except OperationError as e:
-        if space.str_w(w_module) == '__main__':
+        if space.text_w(w_module) == '__main__':
             w_argv = space.sys.getdictvalue(space, 'argv')
             if w_argv and space.len_w(w_argv) > 0:
                 w_filename = space.getitem(w_argv, space.newint(0))
@@ -145,7 +145,7 @@
             check_matched(space, w_mod, w_module) and
             space.abstract_issubclass_w(w_category, w_cat) and
             (ln == 0 or ln == lineno)):
-            return space.str_w(w_action), w_item
+            return space.text_w(w_action), w_item
 
     action = get_default_action(space)
     if not action:
@@ -155,10 +155,10 @@
 def get_default_action(space):
     w_action = get_warnings_attr(space, "defaultaction");
     if w_action is None:
-        return space.str_w(space.fromcache(State).w_default_action)
+        return space.text_w(space.fromcache(State).w_default_action)
 
     space.fromcache(State).w_default_action = w_action
-    return space.str_w(w_action)
+    return space.text_w(w_action)
 
 def get_once_registry(space):
     w_registry = get_warnings_attr(space, "onceregistry");
@@ -188,7 +188,7 @@
     if not space.is_true(w_filename):
         return space.newtext("<unknown>")
 
-    filename = space.str_w(w_filename)
+    filename = space.text_w(w_filename)
     if filename.endswith(".py"):
         n = len(filename) - 3
         assert n >= 0
@@ -201,8 +201,8 @@
     w_stderr = space.sys.get("stderr")
 
     # Print "filename:lineno: category: text\n"
-    message = "%s:%d: %s: %s\n" % (space.str_w(w_filename), lineno,
-                                   space.str_w(w_name), space.str_w(w_text))
+    message = "%s:%d: %s: %s\n" % (space.text_w(w_filename), lineno,
+                                   space.text_w(w_name), space.text_w(w_text))
     space.call_method(w_stderr, "write", space.newtext(message))
 
     # Print "  source_line\n"
@@ -220,7 +220,7 @@
 
     if not w_sourceline:
         return
-    line = space.str_w(w_sourceline)
+    line = space.text_w(w_sourceline)
     if not line:
         return
 
@@ -288,7 +288,7 @@
                 warned = update_registry(space, w_registry, w_text, w_category)
         elif action != 'default':
             try:
-                err = space.str_w(space.str(w_item))
+                err = space.text_w(space.str(w_item))
             except OperationError:
                 err = "???"
             raise oefmt(space.w_RuntimeError,
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -477,15 +477,15 @@
             return space.newtext("array('%s')" % self.typecode)
         elif self.typecode == "c":
             r = space.repr(self.descr_tostring(space))
-            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
             return space.newtext(s)
         elif self.typecode == "u":
             r = space.repr(self.descr_tounicode(space))
-            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
             return space.newtext(s)
         else:
             r = space.repr(self.descr_tolist(space))
-            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
+            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
             return space.newtext(s)
 
 W_ArrayBase.typedef = TypeDef(
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -265,7 +265,8 @@
 
     _exposed_method_names = []
     W_File._decl.im_func(locals(), "bz2__init__",
-                         """Opens a BZ2-compressed file.""")
+                         """Opens a BZ2-compressed file.""",
+                         wrapresult="space.w_None")
     # XXX ^^^ hacking hacking... can't just use the name "__init__" again
     # because the RTyper is confused about the two direct__init__() with
     # a different signature, confusion caused by the fact that
@@ -315,6 +316,7 @@
     **dict([(name, W_File.typedef.rawdict[name])
             for name in same_attributes_as_in_file]))
 
+
 # ____________________________________________________________
 
 def open_bz2file_as_stream(space, w_path, mode="r", buffering=-1,
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -135,7 +135,7 @@
 
     def descr_repr(self, space):
         if self.args_w:
-            args_repr = space.str_w(space.repr(space.newtuple(self.args_w)))
+            args_repr = space.text_w(space.repr(space.newtuple(self.args_w)))
         else:
             args_repr = "()"
         clsname = self.getclass(space).getname(space)
@@ -381,13 +381,13 @@
     def descr_str(self, space):
         if (not space.is_w(self.w_errno, space.w_None) and
             not space.is_w(self.w_strerror, space.w_None)):
-            errno = space.str_w(space.str(self.w_errno))
-            strerror = space.str_w(space.str(self.w_strerror))
+            errno = space.text_w(space.str(self.w_errno))
+            strerror = space.text_w(space.str(self.w_strerror))
             if not space.is_w(self.w_filename, space.w_None):
                 return space.newtext("[Errno %s] %s: %s" % (
                     errno,
                     strerror,
-                    space.str_w(space.repr(self.w_filename))))
+                    space.text_w(space.repr(self.w_filename))))
             return space.newtext("[Errno %s] %s" % (
                 errno,
                 strerror,
@@ -436,10 +436,10 @@
             if not space.is_w(self.w_filename, space.w_None):
                 return space.newtext("[Error %d] %s: %s" % (
                     space.int_w(self.w_winerror),
-                    space.str_w(self.w_strerror),
-                    space.str_w(self.w_filename)))
+                    space.text_w(self.w_strerror),
+                    space.text_w(self.w_filename)))
             return space.newtext("[Error %d] %s" % (space.int_w(self.w_winerror),
-                                                    space.str_w(self.w_strerror)))
+                                                    space.text_w(self.w_strerror)))
         return W_BaseException.descr_str(self, space)
 
     if hasattr(rwin32, 'build_winerror_to_errno'):
@@ -547,7 +547,7 @@
             values_w = space.fixedview(self.args_w[1])
             w_tuple = space.newtuple(values_w + [self.w_lastlineno])
             args_w = [self.args_w[0], w_tuple]
-            args_repr = space.str_w(space.repr(space.newtuple(args_w)))
+            args_repr = space.text_w(space.repr(space.newtuple(args_w)))
             clsname = self.getclass(space).getname(space)
             return space.newtext(clsname + args_repr)
         else:
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -664,7 +664,7 @@
                         pass
                 return w_mod
             elif find_info.modtype == C_EXTENSION and has_so_extension(space):
-                load_c_extension(space, find_info.filename, space.str_w(w_modulename))
+                load_c_extension(space, find_info.filename, space.text_w(w_modulename))
                 return check_sys_modules(space, w_modulename)
         except OperationError:
             w_mods = space.sys.get('modules')
@@ -906,7 +906,7 @@
     """
 
     log_pyverbose(space, 1, "import %s # from %s\n" %
-                  (space.str_w(w_modulename), pathname))
+                  (space.text_w(w_modulename), pathname))
 
     src_stat = os.fstat(fd)
     cpathname = pathname + 'c'
@@ -1030,7 +1030,7 @@
     'sys.modules[modulename]', which must exist.
     """
     log_pyverbose(space, 1, "import %s # compiled from %s\n" %
-                  (space.str_w(w_modulename), cpathname))
+                  (space.text_w(w_modulename), cpathname))
 
     if magic != get_pyc_magic(space):
         raise oefmt(space.w_ImportError, "Bad magic number in %s", cpathname)
@@ -1069,7 +1069,7 @@
     try:
         w_str = space.call_method(w_marshal, 'dumps', co,
                                   space.newint(MARSHAL_VERSION_FOR_PYC))
-        strbuf = space.str_w(w_str)
+        strbuf = space.text_w(w_str)
     except OperationError as e:
         if e.async(space):
             raise
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -79,7 +79,7 @@
     w_suffix, w_filemode, w_modtype = space.unpackiterable(w_info, 3)
 
     filename = space.str0_w(w_filename)
-    filemode = space.str_w(w_filemode)
+    filemode = space.text_w(w_filemode)
     if space.is_w(w_file, space.w_None):
         stream = None
     else:
@@ -89,7 +89,7 @@
         space.int_w(w_modtype),
         filename,
         stream,
-        space.str_w(w_suffix),
+        space.text_w(w_suffix),
         filemode)
     return importing.load_module(
         space, w_name, find_info, reuse=True)
@@ -136,7 +136,7 @@
 def load_dynamic(space, w_modulename, filename, w_file=None):
     if not importing.has_so_extension(space):
         raise oefmt(space.w_ImportError, "Not implemented")
-    importing.load_c_extension(space, filename, space.str_w(w_modulename))
+    importing.load_c_extension(space, filename, space.text_w(w_modulename))
     return importing.check_sys_modules(space, w_modulename)
 
 def new_module(space, w_name):
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -26,11 +26,11 @@
 
     def repr_w(self):
         space = self.space
-        c = space.str_w(space.repr(self.w_c))
+        c = space.text_w(space.repr(self.w_c))
         if self.single_argument():
             s = 'count(%s)' % (c,)
         else:
-            step = space.str_w(space.repr(self.w_step))
+            step = space.text_w(space.repr(self.w_step))
             s = 'count(%s, %s)' % (c, step)
         return self.space.newtext(s)
 
@@ -106,7 +106,7 @@
         return self.space.newint(self.count)
 
     def repr_w(self):
-        objrepr = self.space.str_w(self.space.repr(self.w_obj))
+        objrepr = self.space.text_w(self.space.repr(self.w_obj))
         if self.counting:
             s = 'repeat(%s, %d)' % (objrepr, self.count)
         else:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -103,7 +103,7 @@
     def read(self, n):
         space = self.space
         w_ret = space.call_function(self.func, space.newint(n))
-        ret = space.str_w(w_ret)
+        ret = space.text_w(w_ret)
         if len(ret) != n:
             self.raise_eof()
         return ret
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1222,7 +1222,7 @@
     # XXX slightly non-nice, reuses the sysconf of the underlying os module
     if space.isinstance_w(w_name, space.w_basestring):
         try:
-            num = namespace[space.str_w(w_name)]
+            num = namespace[space.text_w(w_name)]
         except KeyError:
             raise oefmt(space.w_ValueError, "unrecognized configuration name")
     else:
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -658,7 +658,7 @@
         eof = False
         while not eof:
             w_data = space.call_method(w_file, 'read', space.newint(2048))
-            data = space.str_w(w_data)
+            data = space.text_w(w_data)
             eof = len(data) == 0
             w_res = self.Parse(space, data, isfinal=eof)
         return w_res
@@ -674,12 +674,12 @@
         if space.is_w(w_context, space.w_None):
             context = None
         else:
-            context = space.str_w(w_context)
+            context = space.text_w(w_context)
 
         if space.is_none(w_encoding):
             encoding = None
         else:
-            encoding = space.str_w(w_encoding)
+            encoding = space.text_w(w_encoding)
 
         xmlparser = XML_ExternalEntityParserCreate(
             self.itself, context, encoding)
@@ -812,8 +812,8 @@
 Return a new XML parser object."""
     if space.is_none(w_encoding):
         encoding = None
-    elif space.isinstance_w(w_encoding, space.w_str):
-        encoding = space.str_w(w_encoding)
+    elif space.isinstance_w(w_encoding, space.w_text):
+        encoding = space.text_w(w_encoding)
     else:
         raise oefmt(space.w_TypeError,
                     "ParserCreate() argument 1 must be string or None, not %T",
@@ -821,8 +821,8 @@
 
     if space.is_none(w_namespace_separator):
         namespace_separator = 0
-    elif space.isinstance_w(w_namespace_separator, space.w_str):
-        separator = space.str_w(w_namespace_separator)
+    elif space.isinstance_w(w_namespace_separator, space.w_text):
+        separator = space.text_w(w_namespace_separator)
         if len(separator) == 0:
             namespace_separator = 0
         elif len(separator) == 1:
diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py
--- a/pypy/module/sys/interp_encoding.py
+++ b/pypy/module/sys/interp_encoding.py
@@ -10,7 +10,7 @@
 def setdefaultencoding(space, w_encoding):
     """Set the current default string encoding used by the Unicode
 implementation."""
-    encoding = space.str_w(w_encoding)
+    encoding = space.text_w(w_encoding)
     mod = space.getbuiltinmodule("_codecs")
     w_lookup = space.getattr(mod, space.newtext("lookup"))
     # check whether the encoding is there
diff --git a/pypy/module/termios/interp_termios.py b/pypy/module/termios/interp_termios.py
--- a/pypy/module/termios/interp_termios.py
+++ b/pypy/module/termios/interp_termios.py
@@ -28,11 +28,9 @@
     cc = []
     for w_c in space.unpackiterable(w_cc):
         if space.isinstance_w(w_c, space.w_int):
-            ch = space.call_function(space.getattr(w_builtin,
-                                          space.newtext('chr')), w_c)
-            cc.append(space.str_w(ch))
-        else:
-            cc.append(space.str_w(w_c))
+            w_c = space.call_function(space.getattr(w_builtin,
+                                      space.newtext('chr')), w_c)
+        cc.append(space.bytes_w(w_c))
     tup = (space.int_w(w_iflag), space.int_w(w_oflag),
            space.int_w(w_cflag), space.int_w(w_lflag),
            space.int_w(w_ispeed), space.int_w(w_ospeed), cc)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -293,7 +293,7 @@
         for compiled, _, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
                 w_source = self.get_data(space, filename + ext)
-                source = space.str_w(w_source)
+                source = space.bytes_w(w_source)
                 if compiled:
                     magic = importing._get_long(source[:4])
                     timestamp = importing._get_long(source[4:8])
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -93,7 +93,7 @@
 
 class Object(object):
     def descr__getattribute__(space, w_obj, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_descr = space.lookup(w_obj, name)
         if w_descr is not None:
             if space.is_data_descr(w_descr):
@@ -112,7 +112,7 @@
         raiseattrerror(space, w_obj, name)
 
     def descr__setattr__(space, w_obj, w_name, w_value):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_descr = space.lookup(w_obj, name)
         if w_descr is not None:
             if space.is_data_descr(w_descr):
@@ -123,7 +123,7 @@
         raiseattrerror(space, w_obj, name, w_descr)
 
     def descr__delattr__(space, w_obj, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_descr = space.lookup(w_obj, name)
         if w_descr is not None:
             if space.is_data_descr(w_descr):
@@ -860,7 +860,7 @@
             if space.isinstance_w(w_result, space.w_str):
                 return w_result
             try:
-                result = space.str_w(w_result)
+                result = space.str_w(w_result) # YYY
             except OperationError, e:
                 if not e.match(space, space.w_TypeError):
                     raise
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -146,7 +146,7 @@
         return False
 
     def _join_check_item(self, space, w_obj):
-        if (space.isinstance_w(w_obj, space.w_str) or
+        if (space.isinstance_w(w_obj, space.w_bytes) or
             space.isinstance_w(w_obj, space.w_bytearray)):
             return 0
         return 1
@@ -175,7 +175,7 @@
 
     @staticmethod
     def descr_fromhex(space, w_bytearraytype, w_hexstring):
-        hexstring = space.str_w(w_hexstring)
+        hexstring = space.text_w(w_hexstring)
         data = _hexstring_to_array(space, hexstring)
         # in CPython bytearray.fromhex is a staticmethod, so
         # we ignore w_type and always return a bytearray
@@ -484,8 +484,8 @@
 
 
 def getbytevalue(space, w_value):
-    if space.isinstance_w(w_value, space.w_str):
-        string = space.str_w(w_value)
+    if space.isinstance_w(w_value, space.w_bytes):
+        string = space.bytes_w(w_value)
         if len(string) != 1:
             raise oefmt(space.w_ValueError, "string must be of size 1")
         return string[0]
diff --git a/pypy/objspace/std/callmethod.py b/pypy/objspace/std/callmethod.py
--- a/pypy/objspace/std/callmethod.py
+++ b/pypy/objspace/std/callmethod.py
@@ -44,7 +44,7 @@
 
     w_type = space.type(w_obj)
     if w_type.has_object_getattribute():
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         # bit of a mess to use these internal functions, but it allows the
         # mapdict caching below to work without an additional lookup
         version_tag = w_type.version_tag()
@@ -106,7 +106,7 @@
                 break
             w_value = f.popvalue()
             w_key = f.popvalue()
-            key = f.space.str_w(w_key)
+            key = f.space.text_w(w_key)
             keywords[n_kwargs] = key
             keywords_w[n_kwargs] = w_value
 
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -56,8 +56,8 @@
 
     def setitem(self, w_dict, w_key, w_value):
         space = self.space
-        if space.is_w(space.type(w_key), space.w_str):
-            self.setitem_str(w_dict, space.str_w(w_key), w_value)
+        if space.is_w(space.type(w_key), space.w_text):
+            self.setitem_str(w_dict, space.text_w(w_key), w_value)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.setitem(w_key, w_value)
@@ -75,8 +75,8 @@
 
     def setdefault(self, w_dict, w_key, w_default):
         space = self.space
-        if space.is_w(space.type(w_key), space.w_str):
-            key = space.str_w(w_key)
+        if space.is_w(space.type(w_key), space.w_text):
+            key = space.text_w(w_key)
             cell = self.getdictvalue_no_unwrapping(w_dict, key)
             w_result = unwrap_cell(self.space, cell)
             if w_result is not None:
@@ -90,8 +90,8 @@
     def delitem(self, w_dict, w_key):
         space = self.space
         w_key_type = space.type(w_key)
-        if space.is_w(w_key_type, space.w_str):
-            key = space.str_w(w_key)
+        if space.is_w(w_key_type, space.w_text):
+            key = space.text_w(w_key)
             dict_w = self.unerase(w_dict.dstorage)
             try:
                 del dict_w[key]
@@ -111,8 +111,8 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
-            return self.getitem_str(w_dict, space.str_w(w_key))
+        if space.is_w(w_lookup_type, space.w_text):
+            return self.getitem_str(w_dict, space.text_w(w_key))
 
         elif _never_equal_to_string(space, w_lookup_type):
             return None
diff --git a/pypy/objspace/std/classdict.py b/pypy/objspace/std/classdict.py
--- a/pypy/objspace/std/classdict.py
+++ b/pypy/objspace/std/classdict.py
@@ -20,7 +20,7 @@
         w_lookup_type = space.type(w_key)
         if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
             space.abstract_issubclass_w(w_lookup_type, space.w_str)):
-            return self.getitem_str(w_dict, space.str_w(w_key))
+            return self.getitem_str(w_dict, space.text_w(w_key))
         elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
             try:
                 w_key = space.str(w_key)
@@ -29,7 +29,7 @@
                     raise
                 # non-ascii unicode is never equal to a byte string
                 return None
-            return self.getitem_str(w_dict, space.str_w(w_key))
+            return self.getitem_str(w_dict, space.text_w(w_key))
         else:
             return None
 
@@ -39,7 +39,7 @@
     def setitem(self, w_dict, w_key, w_value):
         space = self.space
         if space.is_w(space.type(w_key), space.w_str):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
+            self.setitem_str(w_dict, self.space.text_w(w_key), w_value)
         else:
             raise oefmt(space.w_TypeError,
                         "cannot add non-string keys to dict of a type")
@@ -71,7 +71,7 @@
         space = self.space
         w_key_type = space.type(w_key)
         if space.is_w(w_key_type, space.w_str):
-            key = self.space.str_w(w_key)
+            key = self.space.text_w(w_key)
             if not self.unerase(w_dict.dstorage).deldictvalue(space, key):
                 raise KeyError
         else:
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1418,7 +1418,7 @@
         w_seq = space.call_function(space.w_list, self)
         w_repr = space.repr(w_seq)
         return space.newtext("%s(%s)" % (space.type(self).getname(space),
-                                         space.str_w(w_repr)))
+                                         space.text_w(w_repr)))
 
     def descr_len(self, space):
         return space.len(self.w_dict)
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -39,7 +39,7 @@
 
     def descr_repr(self, space):
         return space.newtext("dict_proxy(%s)" %
-                                (space.str_w(space.repr(self.w_mapping)),))
+                                (space.text_w(space.repr(self.w_mapping)),))
 
     @unwrap_spec(w_default=WrappedDefault(None))
     def get_w(self, space, w_key, w_default):
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -31,7 +31,7 @@
         return _wrapkey(self.space, key)
 
     def unwrap(self, wrapped):
-        return self.space.str_w(wrapped)
+        return self.space.text_w(wrapped)
 
     def get_empty_storage(self):
         d = ([], [])
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -579,9 +579,9 @@
             else:
                 w_obj = space.int(w_obj)
             return newbigint(space, w_longtype, space.bigint_w(w_obj))
-        elif space.isinstance_w(w_value, space.w_str):
+        elif space.isinstance_w(w_value, space.w_bytes):
             return _string_to_w_long(space, w_longtype, w_value,
-                                     space.str_w(w_value))
+                                     space.bytes_w(w_value))
         elif space.isinstance_w(w_value, space.w_unicode):
             from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
             return _string_to_w_long(space, w_longtype, w_value,
@@ -605,7 +605,7 @@
             s = unicode_to_decimal_w(space, w_value)
         else:
             try:
-                s = space.str_w(w_value)
+                s = space.bytes_w(w_value)
             except OperationError:
                 raise oefmt(space.w_TypeError,
                             "long() can't convert non-string with explicit "
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -739,7 +739,7 @@
         space = self.space
         w_lookup_type = space.type(w_key)
         if space.is_w(w_lookup_type, space.w_str):
-            return self.getitem_str(w_dict, space.str_w(w_key))
+            return self.getitem_str(w_dict, space.text_w(w_key))
         elif _never_equal_to_string(space, w_lookup_type):
             return None
         else:
@@ -758,7 +758,7 @@
     def setitem(self, w_dict, w_key, w_value):
         space = self.space
         if space.is_w(space.type(w_key), space.w_str):
-            self.setitem_str(w_dict, self.space.str_w(w_key), w_value)
+            self.setitem_str(w_dict, self.space.text_w(w_key), w_value)
         else:
             self.switch_to_object_strategy(w_dict)
             w_dict.setitem(w_key, w_value)
@@ -766,7 +766,7 @@
     def setdefault(self, w_dict, w_key, w_default):
         space = self.space
         if space.is_w(space.type(w_key), space.w_str):
-            key = space.str_w(w_key)
+            key = space.text_w(w_key)
             w_result = self.getitem_str(w_dict, key)
             if w_result is not None:
                 return w_result
@@ -781,7 +781,7 @@
         w_key_type = space.type(w_key)
         w_obj = self.unerase(w_dict.dstorage)
         if space.is_w(w_key_type, space.w_str):
-            key = self.space.str_w(w_key)
+            key = self.space.text_w(w_key)
             flag = w_obj.deldictvalue(space, key)
             if not flag:
                 raise KeyError
@@ -981,7 +981,7 @@
             return space._handle_getattribute(w_descr, w_obj, w_name)
         version_tag = w_type.version_tag()
         if version_tag is not None:
-            name = space.str_w(w_name)
+            name = space.text_w(w_name)
             # We need to care for obscure cases in which the w_descr is
             # a MutableCell, which may change without changing the version_tag
             _, w_descr = w_type._pure_lookup_where_with_method_cache(
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -161,7 +161,7 @@
         w_module = w_type.lookup("__module__")
         if w_module is not None:
             try:
-                modulename = space.str_w(w_module)
+                modulename = space.text_w(w_module)
             except OperationError as e:
                 if not e.match(space, space.w_TypeError):
                     raise
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -1248,7 +1248,7 @@
         return True
 
     def unwrap(self, w_item):
-        return self.space.str_w(w_item)
+        return self.space.bytes_w(w_item)
 
     def wrap(self, item):
         return self.space.newbytes(item)
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -106,9 +106,9 @@
 
     def descr_repr(self, space):
         return space.newtext("slice(%s, %s, %s)" % (
-            space.str_w(space.repr(self.w_start)),
-            space.str_w(space.repr(self.w_stop)),
-            space.str_w(space.repr(self.w_step))))
+            space.text_w(space.repr(self.w_start)),
+            space.text_w(space.repr(self.w_stop)),
+            space.text_w(space.repr(self.w_step))))
 
     def descr__reduce__(self, space):
         from pypy.objspace.std.sliceobject import W_SliceObject
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1094,11 +1094,13 @@
             return str
         return type(w_obj)
     w_str = str
+    w_text = str
 
     def str_w(self, string):
         assert isinstance(string, str)
         return string
     bytes_w = str_w
+    text_w = str_w
 
     def int_w(self, integer, allow_conversion=True):
         assert isinstance(integer, int)
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -102,8 +102,8 @@
     def descr_repr(self, space):
         items = self.tolist()
         if len(items) == 1:
-            return space.newtext("(" + space.str_w(space.repr(items[0])) + ",)")
-        tmp = ", ".join([space.str_w(space.repr(item)) for item in items])
+            return space.newtext("(" + space.text_w(space.repr(items[0])) + ",)")
+        tmp = ", ".join([space.text_w(space.repr(item)) for item in items])
         return space.newtext("(" + tmp + ")")
 
     def descr_hash(self, space):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -638,7 +638,7 @@
         if w_mod is None or not space.isinstance_w(w_mod, space.w_str):
             mod = None
         else:
-            mod = space.str_w(w_mod)
+            mod = space.text_w(w_mod)
         if not self.is_heaptype():
             kind = 'type'
         else:
@@ -649,7 +649,7 @@
             return space.newtext("<%s '%s'>" % (kind, self.name))
 
     def descr_getattribute(self, space, w_name):
-        name = space.str_w(w_name)
+        name = space.text_w(w_name)
         w_descr = space.lookup(self, name)
         if w_descr is not None:
             if space.is_data_descr(w_descr):
@@ -729,14 +729,14 @@
             return space.call_function(newfunc, w_winner, w_name, w_bases, w_dict)
         w_typetype = w_winner
 
-    name = space.str_w(w_name)
+    name = space.text_w(w_name)
     assert isinstance(name, str)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")
     dict_w = {}
     dictkeys_w = space.listview(w_dict)
     for w_key in dictkeys_w:
-        key = space.str_w(w_key)
+        key = space.text_w(w_key)
         dict_w[key] = space.getitem(w_dict, w_key)
     w_type = space.allocate_instance(W_TypeObject, w_typetype)
     W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
@@ -779,7 +779,7 @@
         raise oefmt(space.w_TypeError,
                     "can only assign string to %N.__name__, not '%T'",
                     w_type, w_value)
-    name = space.str_w(w_value)
+    name = space.text_w(w_value)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")
     w_type.name = name
@@ -1055,7 +1055,7 @@
         else:
             slot_names_w = space.unpackiterable(w_slots)
         for w_slot_name in slot_names_w:
-            slot_name = space.str_w(w_slot_name)
+            slot_name = space.text_w(w_slot_name)
             if slot_name == '__dict__':
                 if wantdict or w_bestbase.hasdict:
                     raise oefmt(space.w_TypeError,
@@ -1106,7 +1106,7 @@
     slot_name = mangle(slot_name, w_self.name)
     if slot_name not in w_self.dict_w:
         # Force interning of slot names.
-        slot_name = space.str_w(space.new_interned_str(slot_name))
+        slot_name = space.text_w(space.new_interned_str(slot_name))
         # in cpython it is ignored less, but we probably don't care
         member = Member(index_next_extra_slot, slot_name, w_self)
         w_self.dict_w[slot_name] = member

From pypy.commits at gmail.com  Wed Nov 30 10:37:22 2016
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 30 Nov 2016 07:37:22 -0800 (PST)
Subject: [pypy-commit] pypy space-newtext: merge default
Message-ID: <583ef232.41a3c20a.7f663.eff1@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: space-newtext
Changeset: r88776:67d4325b8871
Date: 2016-11-30 16:36 +0100
http://bitbucket.org/pypy/pypy/changeset/67d4325b8871/

Log:	merge default

diff too long, truncating to 2000 out of 4915 lines

diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -12,7 +12,6 @@
 
 import sys
 import os
-import shlex
 import imp
 
 from distutils.errors import DistutilsPlatformError
@@ -62,11 +61,31 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
+    g['CC'] = "gcc -pthread"
+    g['CXX'] = "g++ -pthread"
+    g['OPT'] = "-DNDEBUG -O2"
+    g['CFLAGS'] = "-DNDEBUG -O2"
+    g['CCSHARED'] = "-fPIC"
+    g['LDSHARED'] = "gcc -pthread -shared"
+    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+    g['AR'] = "ar"
+    g['ARFLAGS'] = "rc"
     g['EXE'] = ""
-    g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check
-    g['OPT'] = "" 
+    g['VERSION'] = get_python_version()
+
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
 
     global _config_vars
     _config_vars = g
@@ -103,6 +122,12 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
     if args:
         vals = []
         for name in args:
@@ -118,30 +143,80 @@
     """
     return get_config_vars().get(name)
 
+
 def customize_compiler(compiler):
-    """Dummy method to let some easy_install packages that have
-    optional C speedup components.
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile (CPython)
+    or hard-coded in _init_posix() (PyPy).
     """
-    def customize(executable, flags):
-        command = compiler.executables[executable] + flags
-        setattr(compiler, executable, command)
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
 
-    if compiler.compiler_type == "unix":
-        # compiler_so can be c++ which has no -Wimplicit
-        #compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit'])
-        compiler.compiler_so.extend(['-O2', '-fPIC'])
-        compiler.shared_lib_extension = get_config_var('SO')
-        if "CPPFLAGS" in os.environ:
-            cppflags = shlex.split(os.environ["CPPFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cppflags)
-        if "CFLAGS" in os.environ:
-            cflags = shlex.split(os.environ["CFLAGS"])
-            for executable in ('compiler', 'compiler_so', 'linker_so'):
-                customize(executable, cflags)
-        if "LDFLAGS" in os.environ:
-            ldflags = shlex.split(os.environ["LDFLAGS"])
-            customize('linker_so', ldflags)
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
 
 
 from sysconfig_cpython import (
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -20,3 +20,12 @@
 .. branch: clean-exported-state
 
 Clean-ups in the jit optimizeopt
+
+.. branch: conditional_call_value_4
+
+Add jit.conditional_call_elidable(), a way to tell the JIT "conditonally
+call this function" returning a result.
+
+.. branch: desc-specialize
+
+Refactor FunctionDesc.specialize() and related code (RPython annotator).
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -298,6 +298,12 @@
         if config.translation.sandbox:
             config.objspace.lonepycfiles = False
 
+        if config.objspace.usemodules.cpyext:
+            if config.translation.gc != 'incminimark':
+                raise Exception("The 'cpyext' module requires the 'incminimark'"
+                                " GC.  You need either 'targetpypystandalone.py"
+                                " --withoutmod-cpyext' or '--gc=incminimark'")
+
         config.translating = True
 
         import translate
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -111,7 +111,9 @@
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return
+        is_dict = False
         if space.isinstance_w(w_starstararg, space.w_dict):
+            is_dict = True
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -125,7 +127,9 @@
             keys_w = space.unpackiterable(w_keys)
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        _do_combine_starstarargs_wrapped(
+            space, keys_w, w_starstararg, keywords, keywords_w, self.keywords,
+            is_dict)
         self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
@@ -355,7 +359,7 @@
                             key)
 
 def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
-        keywords_w, existingkeywords):
+        keywords_w, existingkeywords, is_dict):
     i = 0
     for w_key in keys_w:
         try:
@@ -374,7 +378,16 @@
                             "got multiple values for keyword argument '%s'",
                             key)
         keywords[i] = key
-        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        if is_dict:
+            # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+            # dict, just ignore the __getitem__ and access the underlying dict
+            # directly
+            from pypy.objspace.descroperation import dict_getitem
+            w_descr = dict_getitem(space)
+            w_value = space.get_and_call_function(w_descr, w_starstararg, w_key)
+        else:
+            w_value = space.getitem(w_starstararg, w_key)
+        keywords_w[i] = w_value
         i += 1
 
 @jit.look_inside_iff(
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -184,6 +184,14 @@
                 assert self._finalize_.im_func is not W_Root._finalize_.im_func
             space.finalizer_queue.register_finalizer(self)
 
+    def may_unregister_rpython_finalizer(self, space):
+        """Optimization hint only: if there is no user-defined __del__()
+        method, pass the hint ``don't call any finalizer'' to rgc.
+        """
+        if not self.getclass(space).hasuserdel:
+            from rpython.rlib import rgc
+            rgc.may_ignore_finalizer(self)
+
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -445,6 +445,7 @@
                                           space.newtext(msg))
         return OperationError(exc, w_error)
 
+ at specialize.arg(3)
 def wrap_oserror2(space, e, w_filename=None, exception_name='w_OSError',
                   w_exception_class=None):
     assert isinstance(e, OSError)
@@ -472,8 +473,8 @@
         w_error = space.call_function(exc, space.newint(errno),
                                       space.newtext(msg))
     return OperationError(exc, w_error)
-wrap_oserror2._annspecialcase_ = 'specialize:arg(3)'
 
+ at specialize.arg(3)
 def wrap_oserror(space, e, filename=None, exception_name='w_OSError',
                  w_exception_class=None):
     if filename is not None:
@@ -484,7 +485,6 @@
         return wrap_oserror2(space, e, None,
                              exception_name=exception_name,
                              w_exception_class=w_exception_class)
-wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
 
 def exception_from_saved_errno(space, w_type):
     from rpython.rlib.rposix import get_saved_errno
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.pyopcode import LoopBlock
 from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 
 
 class GeneratorIterator(W_Root):
@@ -102,11 +102,11 @@
                 w_result = frame.execute_frame(w_arg, operr)
             except OperationError:
                 # errors finish a frame
-                self.frame = None
+                self.frame_is_finished()
                 raise
             # if the frame is now marked as finished, it was RETURNed from
             if frame.frame_finished_execution:
-                self.frame = None
+                self.frame_is_finished()
                 raise OperationError(space.w_StopIteration, space.w_None)
             else:
                 return w_result     # YIELDed
@@ -208,7 +208,7 @@
             finally:
                 frame.f_backref = jit.vref_None
                 self.running = False
-                self.frame = None
+                self.frame_is_finished()
         return unpack_into
     unpack_into = _create_unpack_into()
     unpack_into_w = _create_unpack_into()
@@ -227,6 +227,10 @@
                     break
                 block = block.previous
 
+    def frame_is_finished(self):
+        self.frame = None
+        rgc.may_ignore_finalizer(self)
+
 
 def get_printable_location_genentry(bytecode):
     return '%s <generator>' % (bytecode.get_repr(),)
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -123,6 +123,12 @@
             raise OperationError(AttributeError, name)
         return method(*args)
 
+    def lookup_in_type(self, cls, name):
+        return getattr(cls, name)
+
+    def get_and_call_function(self, w_descr, w_obj, *args):
+        return w_descr.__get__(w_obj)(*args)
+
     def type(self, obj):
         class Type:
             def getname(self, space):
@@ -808,3 +814,19 @@
             assert str(e) == "myerror"
         else:
             assert False, "Expected TypeError"
+
+    def test_dict_subclass_with_weird_getitem(self):
+        # issue 2435: bug-to-bug compatibility with cpython. for a subclass of
+        # dict, just ignore the __getitem__ and behave like ext_do_call in ceval.c
+        # which just uses the underlying dict
+        class d(dict):
+            def __getitem__(self, key):
+                return key
+
+        for key in ["foo", u"foo"]:
+            q = d()
+            q[key] = "bar"
+
+            def test(**kwargs):
+                return kwargs
+            assert test(**q) == {"foo": "bar"}
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -397,7 +397,7 @@
         space = self.space
         if space.is_none(w_destructor):
             if isinstance(self, W_CDataGCP):
-                self.w_destructor = None
+                self.detach_destructor()
                 return space.w_None
             raise oefmt(space.w_TypeError,
                         "Can remove destructor only on a object "
@@ -604,6 +604,10 @@
             self.w_destructor = None
             self.space.call_function(w_destructor, self.w_original_cdata)
 
+    def detach_destructor(self):
+        self.w_destructor = None
+        self.may_unregister_rpython_finalizer(self.space)
+
 
 W_CData.typedef = TypeDef(
     '_cffi_backend.CData',
diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -55,6 +55,7 @@
         if not libhandle:
             raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed",
                         self.libname)
+        self.may_unregister_rpython_finalizer(self.ffi.space)
 
         # Clear the dict to force further accesses to do cdlopen_fetch()
         # again, and fail because the library was closed.  Note that the
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -1,4 +1,5 @@
 import sys
+from rpython.rlib.objectmodel import specialize
 from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
@@ -6,7 +7,6 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.debug import check_nonneg
-from rpython.rlib.objectmodel import specialize
 
 
 # A `dequeobject` is composed of a doubly-linked list of `block` nodes.
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -172,6 +172,7 @@
             self.newlines = self.stream.getnewlines()
             self.stream = None
             self.fd = -1
+            self.may_unregister_rpython_finalizer(self.space)
             openstreams = getopenstreams(self.space)
             try:
                 del openstreams[stream]
@@ -507,11 +508,10 @@
             if space.isinstance_w(w_data, space.w_unicode):
                 # note: "encode" is called before we acquire the lock
                 # for this file, which is done in file_write_str().
-                # Direct access to unicodeobject because we don't want
+                # Use a specific space method because we don't want
                 # to call user-defined "encode" methods here.
-                from pypy.objspace.std.unicodeobject import encode_object
-                w_data = encode_object(space, w_data, self.encoding,
-                                       self.errors)
+                w_data = space.encode_unicode_object(w_data,
+                     self.encoding, self.errors)
             data = space.charbuf_w(w_data)
         self.file_write_str(data)
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -233,6 +233,7 @@
         except SocketError:
             # cpython doesn't return any errors on close
             pass
+        self.may_unregister_rpython_finalizer(space)
 
     def connect_w(self, space, w_addr):
         """connect(address)
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -217,7 +217,7 @@
             return self.space.w_None
         return w_obj
 
-    def descr__eq__(self, space, w_ref2):
+    def compare(self, space, w_ref2, invert):
         if not isinstance(w_ref2, W_Weakref):
             return space.w_NotImplemented
         ref1 = self
@@ -225,11 +225,18 @@
         w_obj1 = ref1.dereference()
         w_obj2 = ref2.dereference()
         if w_obj1 is None or w_obj2 is None:
-            return space.is_(ref1, ref2)
-        return space.eq(w_obj1, w_obj2)
+            w_res = space.is_(ref1, ref2)
+        else:
+            w_res = space.eq(w_obj1, w_obj2)
+        if invert:
+            w_res = space.not_(w_res)
+        return w_res
+
+    def descr__eq__(self, space, w_ref2):
+        return self.compare(space, w_ref2, invert=False)
 
     def descr__ne__(self, space, w_ref2):
-        return space.not_(space.eq(self, w_ref2))
+        return self.compare(space, w_ref2, invert=True)
 
 def getlifeline(space, w_obj):
     lifeline = w_obj.getweakref()
diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py
--- a/pypy/module/_weakref/test/test_weakref.py
+++ b/pypy/module/_weakref/test/test_weakref.py
@@ -150,6 +150,14 @@
         assert not (ref1 == [])
         assert ref1 != []
 
+    def test_ne(self):
+        import _weakref
+        class X(object):
+            pass
+        ref1 = _weakref.ref(X())
+        assert ref1.__eq__(X()) is NotImplemented
+        assert ref1.__ne__(X()) is NotImplemented
+
     def test_getweakrefs(self):
         import _weakref, gc
         class A(object):
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -291,7 +291,7 @@
 same_attributes_as_in_file.remove('__init__')
 same_attributes_as_in_file.extend([
     'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace',
-    'writelines', '__exit__', '__weakref__'])
+    'writelines', '__exit__', '__weakref__', 'write'])
 
 W_BZ2File.typedef = TypeDef(
     "BZ2File",
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -189,13 +189,13 @@
         assert isinstance(w_obj, FakeFloat)
         return w_obj.val
 
+    @specialize.arg(1)
     def interp_w(self, RequiredClass, w_obj, can_be_None=False):
         if can_be_None and w_obj is None:
             return None
         if not isinstance(w_obj, RequiredClass):
             raise TypeError
         return w_obj
-    interp_w._annspecialcase_ = 'specialize:arg(1)'
 
     def getarg_w(self, code, w_obj):    # for retrieving buffers
         return FakeBuffer(w_obj)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -36,8 +36,6 @@
 from rpython.rlib.objectmodel import specialize
 from pypy.module import exceptions
 from pypy.module.exceptions import interp_exceptions
-# CPython 2.4 compatibility
-from py.builtin import BaseException
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib import rawrefcount
@@ -985,7 +983,7 @@
         py_type_ready(space, get_capsule_type())
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
-    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], 
+    _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
                                   lltype.Void, compilation_info=eci)
     def reinit_tls(space):
         _reinit_tls()
@@ -1614,9 +1612,8 @@
     miniglobals = {'__name__':    __name__, # for module name propagation
                    }
     exec source.compile() in miniglobals
-    call_external_function = miniglobals['cpy_call_external']
+    call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
     call_external_function._dont_inline_ = True
-    call_external_function._annspecialcase_ = 'specialize:ll'
     call_external_function._gctransformer_hint_close_stack_ = True
     # don't inline, as a hack to guarantee that no GC pointer is alive
     # anywhere in call_external_function
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -137,8 +137,7 @@
     """This is the same as PyDict_Merge(a, b, 1) in C, or a.update(b) in
     Python.  Return 0 on success or -1 if an exception was raised.
     """
-    space.call_method(space.w_dict, "update", w_obj, w_other)
-    return 0
+    return PyDict_Merge(space, w_obj, w_other, 1)
 
 @cpython_api([PyObject], PyObject)
 def PyDict_Keys(space, w_obj):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -62,12 +62,14 @@
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
+    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.ensure_object_strategy()  # make sure we can return a borrowed obj
-    # XXX ^^^ how does this interact with CPyListStrategy?
+    cpy_strategy = space.fromcache(CPyListStrategy)
+    if w_list.strategy is not cpy_strategy:
+        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
     w_res = w_list.getitem(index)
     return w_res     # borrowed ref
 
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import cpython_api, cpython_struct, \
         METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING
-from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.pyobject import PyObject, as_pyobj
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
     W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod,
@@ -124,11 +124,17 @@
     else:
         PyErr_BadInternalCall(space)
 
- at cpython_api([PyObject], rffi.CCHARP, error=0)
-def PyModule_GetName(space, module):
+ at cpython_api([PyObject], rffi.CCHARP)
+def PyModule_GetName(space, w_mod):
     """
     Return module's __name__ value.  If the module does not provide one,
-    or if it is not a string, SystemError is raised and NULL is returned."""
-    raise NotImplementedError
-
-
+    or if it is not a string, SystemError is raised and NULL is returned.
+    """
+    # NOTE: this version of the code works only because w_mod.w_name is
+    # a wrapped string object attached to w_mod; so it makes a
+    # PyStringObject that will live as long as the module itself,
+    # and returns a "char *" inside this PyStringObject.
+    if not isinstance(w_mod, Module):
+        raise oefmt(space.w_SystemError, "PyModule_GetName(): not a module")
+    from pypy.module.cpyext.bytesobject import PyString_AsString
+    return PyString_AsString(space, as_pyobj(space, w_mod.w_name))
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -252,7 +252,10 @@
 def PyObject_Format(space, w_obj, w_format_spec):
     if w_format_spec is None:
         w_format_spec = space.newtext('')
-    return space.call_method(w_obj, '__format__', w_format_spec)
+    w_ret = space.call_method(w_obj, '__format__', w_format_spec)
+    if space.isinstance_w(w_format_spec, space.w_unicode):
+        return space.unicode_from_object(w_ret)
+    return w_ret
 
 @cpython_api([PyObject], PyObject)
 def PyObject_Unicode(space, w_obj):
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,7 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import fatalerror
 
 
 #________________________________________________________
@@ -192,6 +193,8 @@
     rawrefcount.create_link_pypy(w_obj, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 def from_ref(space, ref):
     """
     Finds the interpreter object corresponding to the given reference.  If the
@@ -202,7 +205,23 @@
         return None
     w_obj = rawrefcount.to_obj(W_Root, ref)
     if w_obj is not None:
-        return w_obj
+        if w_obj is not w_marker_deallocating:
+            return w_obj
+        fatalerror(
+            "*** Invalid usage of a dying CPython object ***\n"
+            "\n"
+            "cpyext, the emulation layer, detected that while it is calling\n"
+            "an object's tp_dealloc, the C code calls back a function that\n"
+            "tries to recreate the PyPy version of the object.  Usually it\n"
+            "means that tp_dealloc calls some general PyXxx() API.  It is\n"
+            "a dangerous and potentially buggy thing to do: even in CPython\n"
+            "the PyXxx() function could, in theory, cause a reference to the\n"
+            "object to be taken and stored somewhere, for an amount of time\n"
+            "exceeding tp_dealloc itself.  Afterwards, the object will be\n"
+            "freed, making that reference point to garbage.\n"
+            ">>> PyPy could contain some workaround to still work if\n"
+            "you are lucky, but it is not done so far; better fix the bug in\n"
+            "the CPython extension.")
 
     # This reference is not yet a real interpreter object.
     # Realize it.
@@ -233,7 +252,8 @@
 INTERPLEVEL_API['as_pyobj'] = as_pyobj
 
 def pyobj_has_w_obj(pyobj):
-    return rawrefcount.to_obj(W_Root, pyobj) is not None
+    w_obj = rawrefcount.to_obj(W_Root, pyobj)
+    return w_obj is not None and w_obj is not w_marker_deallocating
 INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj)
 
 
@@ -335,6 +355,7 @@
     pto = obj.c_ob_type
     #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \
     #      "'s type which is", rffi.charp2str(pto.c_tp_name)
+    rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -103,6 +103,17 @@
         api.PyDict_Update(w_d, w_d2)
         assert space.unwrap(w_d) == dict(a='b', c='d', e='f')
 
+    def test_update_doesnt_accept_list_of_tuples(self, space, api):
+        w_d = space.newdict()
+        space.setitem(w_d, space.wrap("a"), space.wrap("b"))
+
+        w_d2 = space.wrap([("c", "d"), ("e", "f")])
+
+        api.PyDict_Update(w_d, w_d2)
+        assert api.PyErr_Occurred() is space.w_AttributeError
+        api.PyErr_Clear()
+        assert space.unwrap(w_d) == dict(a='b') # unchanged
+
     def test_iter(self, space, api):
         w_dict = space.sys.getdict(space)
         py_dict = make_ref(space, w_dict)
@@ -199,3 +210,18 @@
              """),
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
+
+    def test_update(self):
+        module = self.import_extension('foo', [
+            ("update", "METH_VARARGS",
+             '''
+             if (PyDict_Update(PyTuple_GetItem(args, 0), PyTuple_GetItem(args, 1)))
+                return NULL;
+             Py_RETURN_NONE;
+             ''')])
+        d = {"a": 1}
+        module.update(d, {"c": 2})
+        assert d == dict(a=1, c=2)
+        d = {"a": 1}
+        raises(AttributeError, module.update, d, [("c", 2)])
+
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_module.py
@@ -0,0 +1,12 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi
+
+
+class TestModuleObject(BaseApiTest):
+    def test_module_getname(self, space, api):
+        w_sys = space.wrap(space.sys)
+        p = api.PyModule_GetName(w_sys)
+        assert rffi.charp2str(p) == 'sys'
+        p2 = api.PyModule_GetName(w_sys)
+        assert p2 == p
+        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -312,6 +312,16 @@
         assert isinstance(dict(), collections.Mapping)
         assert module.ismapping(dict())
 
+    def test_format_returns_unicode(self):
+        module = self.import_extension('foo', [
+            ("empty_format", "METH_O",
+            """
+                PyObject* empty_unicode = PyUnicode_FromStringAndSize("", 0);
+                PyObject* obj = PyObject_Format(args, empty_unicode);
+                return obj;
+            """)])
+        a = module.empty_format('hello')
+        assert isinstance(a, unicode)
 
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -331,12 +331,34 @@
                  PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
                  Py_INCREF(heaptype->ht_name);
                  return heaptype->ht_name;
+             '''),
+            ("setattr", "METH_O",
              '''
-             )
+                int ret;
+                PyObject* name = PyString_FromString("mymodule");
+                PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
+                PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
+                if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
+                {
+                    PyErr_SetString(PyExc_ValueError,
+                                    "Py_TPFLAGS_HEAPTYPE not set");
+                    return NULL;
+                }
+                type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
+                PyType_Ready(&type->ht_type);
+                ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
+                                    "__module__", name);
+                Py_DECREF(name);
+                if (ret < 0)
+                    return NULL;
+                return PyLong_FromLong(ret);
+             '''),
             ])
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
+        assert module.setattr(C) == 0
+
 
     def test_type_dict(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -471,7 +471,7 @@
         W_TypeObject.__init__(self, space, name,
             bases_w or [space.w_object], dict_w, force_new_layout=new_layout)
         self.flag_cpytype = True
-        self.flag_heaptype = False
+        self.flag_heaptype = pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
         # if a sequence or a mapping, then set the flag to force it
         if pto.c_tp_as_sequence and pto.c_tp_as_sequence.c_sq_item:
             self.flag_map_or_seq = 'S'
@@ -854,14 +854,14 @@
     w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
     track_reference(space, py_obj, w_obj)
     # __init__ wraps all slotdefs functions from py_type via add_operators
-    w_obj.__init__(space, py_type) 
+    w_obj.__init__(space, py_type)
     w_obj.ready()
 
     finish_type_2(space, py_type, w_obj)
     base = py_type.c_tp_base
     if base:
         # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots
-        if not py_type.c_tp_as_number: 
+        if not py_type.c_tp_as_number:
             py_type.c_tp_as_number = base.c_tp_as_number
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
             py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,4 +1,5 @@
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():
+    import pypy.module._vmprof.interp_vmprof   # register_code_object_class()
     checkmodule('faulthandler')
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -307,7 +307,8 @@
     def __init__(self, space):
         self.stat_float_times = True
 
-def stat_float_times(space, w_value=None):
+ at unwrap_spec(newval=int)
+def stat_float_times(space, newval=-1):
     """stat_float_times([newval]) -> oldval
 
 Determine whether os.[lf]stat represents time stamps as float objects.
@@ -317,10 +318,10 @@
 """
     state = space.fromcache(StatState)
 
-    if w_value is None:
+    if newval == -1:
         return space.newbool(state.stat_float_times)
     else:
-        state.stat_float_times = space.bool_w(w_value)
+        state.stat_float_times = (newval != 0)
 
 
 @unwrap_spec(fd=c_int)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -159,11 +159,14 @@
             st = posix.stat(path)
             assert isinstance(st.st_mtime, float)
             assert st[7] == int(st.st_atime)
+            assert posix.stat_float_times(-1) is True
 
             posix.stat_float_times(False)
             st = posix.stat(path)
             assert isinstance(st.st_mtime, (int, long))
             assert st[7] == st.st_atime
+            assert posix.stat_float_times(-1) is False
+
         finally:
             posix.stat_float_times(current)
 
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -3,11 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib import rgc, jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform
-from rpython.rlib.objectmodel import specialize
 
 import sys
 import weakref
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -67,7 +67,8 @@
             p10 = call_r(ConstClass(ll_str__IntegerR_SignedConst_Signed), i5, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
             guard_nonnull(p10, descr=...)
-            i12 = call_i(ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=0>)
+            i99 = strhash(p10)
+            i12 = cond_call_value_i(i99, ConstClass(_ll_strhash__rpy_stringPtr), p10, descr=<Calli . r EF=2>)
             p13 = new(descr=...)
             p15 = new_array_clear(16, descr=<ArrayU 1>)
             {{{
@@ -86,6 +87,7 @@
             call_n(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<Callv 0 rrrii EF=5>)
             setfield_gc(p20, i5, descr=<FieldS .*W_IntObject.inst_intval .* pure>)
             guard_no_exception(descr=...)
+            i98 = strhash(p10)
             i23 = call_i(ConstClass(ll_call_lookup_function), p13, p10, i12, 0, descr=<Calli . rrii EF=5 OS=4>)
             guard_no_exception(descr=...)
             i27 = int_lt(i23, 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -387,7 +387,8 @@
     def test_long_comparison(self):
         def main(n):
             while n:
-                12345L > 123L  # ID: long_op
+                x = 12345L
+                x > 123L  # ID: long_op
                 n -= 1
 
         log = self.run(main, [300])
diff --git a/pypy/module/select/interp_epoll.py b/pypy/module/select/interp_epoll.py
--- a/pypy/module/select/interp_epoll.py
+++ b/pypy/module/select/interp_epoll.py
@@ -79,6 +79,7 @@
 
 class W_Epoll(W_Root):
     def __init__(self, space, epfd):
+        self.space = space
         self.epfd = epfd
         self.register_finalizer(space)
 
@@ -113,6 +114,7 @@
         if not self.get_closed():
             socketclose(self.epfd)
             self.epfd = -1
+            self.may_unregister_rpython_finalizer(self.space)
 
     def epoll_ctl(self, space, ctl, w_fd, eventmask, ignore_ebadf=False):
         fd = space.c_filedescriptor_w(w_fd)
diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py
--- a/pypy/module/select/interp_kqueue.py
+++ b/pypy/module/select/interp_kqueue.py
@@ -108,6 +108,7 @@
 
 class W_Kqueue(W_Root):
     def __init__(self, space, kqfd):
+        self.space = space
         self.kqfd = kqfd
         self.register_finalizer(space)
 
@@ -132,6 +133,7 @@
             kqfd = self.kqfd
             self.kqfd = -1
             socketclose_no_errno(kqfd)
+            self.may_unregister_rpython_finalizer(self.space)
 
     def check_closed(self, space):
         if self.get_closed():
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -202,6 +202,7 @@
                 if mode == rzlib.Z_FINISH:    # release the data structures now
                     rzlib.deflateEnd(self.stream)
                     self.stream = rzlib.null_stream
+                    self.may_unregister_rpython_finalizer(space)
             finally:
                 self.unlock()
         except rzlib.RZlibError as e:
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -12,65 +12,53 @@
 @specialize.memo()
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
-    w_src, w_getattribute = space.lookup_in_type_where(space.w_object,
-                                                       '__getattribute__')
-    return w_getattribute
+    return space.lookup_in_type(space.w_object, '__getattribute__')
 
 @specialize.memo()
 def object_setattr(space):
     "Utility that returns the app-level descriptor object.__setattr__."
-    w_src, w_setattr = space.lookup_in_type_where(space.w_object,
-                                                  '__setattr__')
-    return w_setattr
+    return space.lookup_in_type(space.w_object, '__setattr__')
 
 @specialize.memo()
 def object_delattr(space):
     "Utility that returns the app-level descriptor object.__delattr__."
-    w_src, w_delattr = space.lookup_in_type_where(space.w_object,
-                                                  '__delattr__')
-    return w_delattr
+    return space.lookup_in_type(space.w_object, '__delattr__')
 
 @specialize.memo()
 def object_hash(space):
     "Utility that returns the app-level descriptor object.__hash__."
-    w_src, w_hash = space.lookup_in_type_where(space.w_object,
-                                                  '__hash__')
-    return w_hash
+    return space.lookup_in_type(space.w_object, '__hash__')
 
 @specialize.memo()
 def type_eq(space):
     "Utility that returns the app-level descriptor type.__eq__."
-    w_src, w_eq = space.lookup_in_type_where(space.w_type,
-                                             '__eq__')
-    return w_eq
+    return space.lookup_in_type(space.w_type, '__eq__')
 
 @specialize.memo()
 def list_iter(space):
     "Utility that returns the app-level descriptor list.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_list,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_list, '__iter__')
 
 @specialize.memo()
 def tuple_iter(space):
     "Utility that returns the app-level descriptor tuple.__iter__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_tuple,
-                                               '__iter__')
-    return w_iter
+    return space.lookup_in_type(space.w_tuple, '__iter__')
 
 @specialize.memo()
 def str_getitem(space):
     "Utility that returns the app-level descriptor str.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_str,
-                                               '__getitem__')
-    return w_iter
+    return space.lookup_in_type(space.w_str, '__getitem__')
 
 @specialize.memo()
 def unicode_getitem(space):
     "Utility that returns the app-level descriptor unicode.__getitem__."
-    w_src, w_iter = space.lookup_in_type_where(space.w_unicode,
-                                               '__getitem__')
-    return w_iter
+    return space.lookup_in_type(space.w_unicode, '__getitem__')
+
+ at specialize.memo()
+def dict_getitem(space):
+    "Utility that returns the app-level descriptor dict.__getitem__."
+    return space.lookup_in_type(space.w_dict, '__getitem__')
+
 
 def raiseattrerror(space, w_obj, name, w_descr=None):
     if w_descr is None:
@@ -865,7 +853,7 @@
                 if not e.match(space, space.w_TypeError):
                     raise
                 raise oefmt(space.w_TypeError,
-                            "%(specialname)s returned non-%(targetname)s "
+                            "%(specialname)s returned non-string "
                             "(type '%%T')", w_result)
             else:
                 # re-wrap the result as a real string
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -66,15 +66,16 @@
 class W_MyType(W_MyObject):
     name = "foobar"
     flag_map_or_seq = '?'
+    hasuserdel = False
 
     def __init__(self):
         self.mro_w = [w_some_obj(), w_some_obj()]
         self.dict_w = {'__str__': w_some_obj()}
+        self.hasuserdel = True
 
     def get_module(self):
         return w_some_obj()
 
-
     def getname(self, space):
         return self.name
 
@@ -206,6 +207,7 @@
     newtext = newbytes
     newtext_or_none = newbytes
 
+    @specialize.argtype(1)
     def wrap(self, x):
         if not we_are_translated():
             if isinstance(x, gateway.interp2app):
@@ -219,7 +221,6 @@
                 return w_some_obj()
             self._wrap_not_rpython(x)
         return w_some_obj()
-    wrap._annspecialcase_ = "specialize:argtype(1)"
 
     def _wrap_not_rpython(self, x):
         "NOT_RPYTHON"
@@ -309,10 +310,10 @@
         is_root(w_complex)
         return 1.1, 2.2
 
+    @specialize.arg(1)
     def allocate_instance(self, cls, w_subtype):
         is_root(w_subtype)
         return instantiate(cls)
-    allocate_instance._annspecialcase_ = "specialize:arg(1)"
 
     def decode_index(self, w_index_or_slice, seqlength):
         is_root(w_index_or_slice)
@@ -334,12 +335,18 @@
     def unicode_from_object(self, w_obj):
         return w_some_obj()
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        return w_some_obj()
+
     def _try_fetch_pycode(self, w_func):
         return None
 
     def is_generator(self, w_obj):
         return NonConstant(False)
 
+    def lookup_in_type(self, w_type, name):
+        return w_some_obj()
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/fake/test/test_checkmodule.py b/pypy/objspace/fake/test/test_checkmodule.py
--- a/pypy/objspace/fake/test/test_checkmodule.py
+++ b/pypy/objspace/fake/test/test_checkmodule.py
@@ -9,9 +9,9 @@
 
 def make_checker():
     check = []
+    @specialize.memo()
     def see():
         check.append(True)
-    see._annspecialcase_ = 'specialize:memo'
     return see, check
 
 def test_wrap_interp2app():
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -2,12 +2,12 @@
 import sys
 
 from rpython.rlib import jit
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import INT_MAX
 from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import specialize
 
 from pypy.interpreter.error import OperationError, oefmt
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -345,6 +345,10 @@
         return w_type.lookup(name)
     lookup._annspecialcase_ = 'specialize:lookup'
 
+    def lookup_in_type(self, w_type, name):
+        w_src, w_descr = self.lookup_in_type_where(w_type, name)
+        return w_descr
+
     def lookup_in_type_where(self, w_type, name):
         return w_type.lookup_where(name)
     lookup_in_type_where._annspecialcase_ = 'specialize:lookup_in_type_where'
@@ -632,6 +636,10 @@
         from pypy.objspace.std.unicodeobject import unicode_from_object
         return unicode_from_object(self, w_obj)
 
+    def encode_unicode_object(self, w_unicode, encoding, errors):
+        from pypy.objspace.std.unicodeobject import encode_object
+        return encode_object(self, w_unicode, encoding, errors)
+
     def call_method(self, w_obj, methname, *arg_w):
         return callmethod.call_method_opt(self, w_obj, methname, *arg_w)
 
@@ -661,4 +669,4 @@
     @specialize.arg(2, 3)
     def is_overloaded(self, w_obj, tp, method):
         return (self.lookup(w_obj, method) is not
-                self.lookup_in_type_where(tp, method)[1])
+                self.lookup_in_type(tp, method))
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -315,7 +315,8 @@
                 assert operate(A()) == "world" * n
             assert type(operate(A())) is str
             answer = 42
-            raises(TypeError, operate, A())
+            excinfo = raises(TypeError, operate, A())
+            assert "returned non-string (type 'int')" in str(excinfo.value)
 
     def test_missing_getattribute(self):
         class X(object):
diff --git a/pypy/tool/cpyext/extbuild.py b/pypy/tool/cpyext/extbuild.py
--- a/pypy/tool/cpyext/extbuild.py
+++ b/pypy/tool/cpyext/extbuild.py
@@ -204,6 +204,10 @@
         pass
     from distutils.ccompiler import new_compiler
     from distutils import sysconfig
+
+    # XXX for Darwin running old versions of CPython 2.7.x
+    sysconfig.get_config_vars()
+
     compiler = new_compiler(force=1)
     sysconfig.customize_compiler(compiler)  # XXX
     objects = []
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -2,6 +2,7 @@
 
 import types
 from collections import defaultdict
+from contextlib import contextmanager
 
 from rpython.tool.ansi_print import AnsiLogger
 from rpython.tool.pairtype import pair
@@ -22,7 +23,8 @@
     """Block annotator for RPython.
     See description in doc/translation.txt."""
 
-    def __init__(self, translator=None, policy=None, bookkeeper=None):
+    def __init__(self, translator=None, policy=None, bookkeeper=None,
+            keepgoing=False):
         import rpython.rtyper.extfuncregistry # has side effects
 
         if translator is None:
@@ -50,6 +52,9 @@
         if bookkeeper is None:
             bookkeeper = Bookkeeper(self)
         self.bookkeeper = bookkeeper
+        self.keepgoing = keepgoing
+        self.failed_blocks = set()
+        self.errors = []
 
     def __getstate__(self):
         attrs = """translator pendingblocks annotated links_followed
@@ -79,22 +84,17 @@
         annmodel.TLS.check_str_without_nul = (
             self.translator.config.translation.check_str_without_nul)
 
-        flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
+        with self.using_policy(policy):
+            flowgraph, inputs_s = self.get_call_parameters(function, args_s)
 
         if main_entry_point:
             self.translator.entry_point_graph = flowgraph
         return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
 
-    def get_call_parameters(self, function, args_s, policy):
-        desc = self.bookkeeper.getdesc(function)
-        prevpolicy = self.policy
-        self.policy = policy
-        self.bookkeeper.enter(None)
-        try:
+    def get_call_parameters(self, function, args_s):
+        with self.bookkeeper.at_position(None):
+            desc = self.bookkeeper.getdesc(function)
             return desc.get_call_parameters(args_s)
-        finally:
-            self.bookkeeper.leave()
-            self.policy = prevpolicy
 
     def annotate_helper(self, function, args_s, policy=None):
         if policy is None:
@@ -103,21 +103,29 @@
             # XXX hack
             annmodel.TLS.check_str_without_nul = (
                 self.translator.config.translation.check_str_without_nul)
-        graph, inputcells = self.get_call_parameters(function, args_s, policy)
-        self.build_graph_types(graph, inputcells, complete_now=False)
-        self.complete_helpers(policy)
+        with self.using_policy(policy):
+            graph, inputcells = self.get_call_parameters(function, args_s)
+            self.build_graph_types(graph, inputcells, complete_now=False)
+            self.complete_helpers()
         return graph
 
-    def complete_helpers(self, policy):
-        saved = self.policy, self.added_blocks
-        self.policy = policy
+    def complete_helpers(self):
+        saved = self.added_blocks
+        self.added_blocks = {}
         try:
-            self.added_blocks = {}
             self.complete()
             # invoke annotation simplifications for the new blocks
             self.simplify(block_subset=self.added_blocks)
         finally:
-            self.policy, self.added_blocks = saved
+            self.added_blocks = saved
+
+    @contextmanager
+    def using_policy(self, policy):
+        """A context manager that temporarily replaces the annotator policy"""
+        old_policy = self.policy
+        self.policy = policy
+        yield
+        self.policy = old_policy
 
     def build_graph_types(self, flowgraph, inputcells, complete_now=True):
         checkgraph(flowgraph)
@@ -202,6 +210,12 @@
         else:
             newgraphs = self.translator.graphs  #all of them
             got_blocked_blocks = False in self.annotated.values()
+        if self.failed_blocks:
+            text = ('Annotation failed, %s errors were recorded:' %
+                    len(self.errors))
+            text += '\n-----'.join(str(e) for e in self.errors)
+            raise annmodel.AnnotatorError(text)
+
         if got_blocked_blocks:
             for graph in self.blocked_graphs.values():
                 self.blocked_graphs[graph] = True
@@ -348,6 +362,8 @@
 
         #print '* processblock', block, cells
         self.annotated[block] = graph
+        if block in self.failed_blocks:
+            return
         if block in self.blocked_blocks:
             del self.blocked_blocks[block]
         try:
@@ -392,6 +408,10 @@
         except annmodel.UnionError as e:
             # Add source code to the UnionError
             e.source = '\n'.join(source_lines(graph, block, None, long=True))
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
         # if the merged cells changed, we must redo the analysis
         if unions != oldcells:
@@ -482,6 +502,10 @@
 
         except annmodel.AnnotatorError as e: # note that UnionError is a subclass
             e.source = gather_error(self, graph, block, i)
+            if self.keepgoing:
+                self.errors.append(e)
+                self.failed_blocks.add(block)
+                return
             raise
 
         else:
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -9,6 +9,7 @@
 from collections import OrderedDict
 
 from rpython.flowspace.model import Constant
+from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.model import (
     SomeOrderedDict, SomeString, SomeChar, SomeFloat, unionof, SomeInstance,
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
@@ -21,6 +22,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
+from rpython.annotator.specialize import memo
 from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
@@ -358,7 +360,7 @@
             return self.descs[obj_key]
         except KeyError:
             if isinstance(pyobj, types.FunctionType):
-                result = description.FunctionDesc(self, pyobj)
+                result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
                     raise Exception("ClassDesc for object not supported")
@@ -403,6 +405,23 @@
             self.descs[obj_key] = result
             return result
 
+    def newfuncdesc(self, pyfunc):
+        name = pyfunc.__name__
+        if hasattr(pyfunc, '_generator_next_method_of_'):
+            from rpython.flowspace.argument import Signature
+            signature = Signature(['entry'])     # haaaaaack
+            defaults = ()
+        else:
+            signature = cpython_code_signature(pyfunc.func_code)
+            defaults = pyfunc.func_defaults
+        # get the specializer based on the tag of the 'pyobj'
+        # (if any), according to the current policy
+        tag = getattr(pyfunc, '_annspecialcase_', None)
+        specializer = self.annotator.policy.get_specializer(tag)
+        if specializer is memo:
+            return description.MemoDesc(self, pyfunc, name, signature, defaults, specializer)
+        return description.FunctionDesc(self, pyfunc, name, signature, defaults, specializer)
+
     def getfrozen(self, pyobj):
         return description.FrozenDesc(self, pyobj)
 
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -608,7 +608,7 @@
             if mixin:
                 # make a new copy of the FunctionDesc for this class,
                 # but don't specialize further for all subclasses
-                funcdesc = FunctionDesc(self.bookkeeper, value)
+                funcdesc = self.bookkeeper.newfuncdesc(value)
                 self.classdict[name] = funcdesc
                 return
             # NB. if value is, say, AssertionError.__init__, then we
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -3,11 +3,10 @@
 from rpython.annotator.signature import (
     enforce_signature_args, enforce_signature_return, finish_type)
 from rpython.flowspace.model import FunctionGraph
-from rpython.flowspace.bytecode import cpython_code_signature
 from rpython.annotator.argument import rawshape, ArgErr, simple_args
 from rpython.tool.sourcetools import valid_identifier
 from rpython.tool.pairtype import extendabletype
-from rpython.annotator.model import AnnotatorError, s_ImpossibleValue
+from rpython.annotator.model import AnnotatorError, s_ImpossibleValue, unionof
 
 class CallFamily(object):
     """A family of Desc objects that could be called from common call sites.
@@ -117,7 +116,6 @@
         self.s_value = s_ImpossibleValue    # union of possible values
 
     def update(self, other):
-        from rpython.annotator.model import unionof
         self.descs.update(other.descs)
         self.read_locations.update(other.read_locations)
         self.s_value = unionof(self.s_value, other.s_value)
@@ -192,24 +190,12 @@
 class FunctionDesc(Desc):
     knowntype = types.FunctionType
 
-    def __init__(self, bookkeeper, pyobj=None,
-                 name=None, signature=None, defaults=None,
+    def __init__(self, bookkeeper, pyobj, name, signature, defaults,
                  specializer=None):
         super(FunctionDesc, self).__init__(bookkeeper, pyobj)
-        if name is None:
-            name = pyobj.func_name
-        if signature is None:
-            if hasattr(pyobj, '_generator_next_method_of_'):
-                from rpython.flowspace.argument import Signature
-                signature = Signature(['entry'])     # haaaaaack
-                defaults = ()
-            else:
-                signature = cpython_code_signature(pyobj.func_code)
-        if defaults is None:
-            defaults = pyobj.func_defaults
         self.name = name
         self.signature = signature
-        self.defaults = defaults or ()
+        self.defaults = defaults if defaults is not None else ()
         # 'specializer' is a function with the following signature:
         #      specializer(funcdesc, args_s) => graph
         #                                 or => s_result (overridden/memo cases)
@@ -288,12 +274,43 @@
                 getattr(self.bookkeeper, "position_key", None) is not None):
             _, block, i = self.bookkeeper.position_key
             op = block.operations[i]
-        if self.specializer is None:
-            # get the specializer based on the tag of the 'pyobj'
-            # (if any), according to the current policy
-            tag = getattr(self.pyobj, '_annspecialcase_', None)
-            policy = self.bookkeeper.annotator.policy
-            self.specializer = policy.get_specializer(tag)
+        self.normalize_args(inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
+
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        graph = self.specialize(inputcells, op)
+        assert isinstance(graph, FunctionGraph)
+        # if that graph has a different signature, we need to re-parse
+        # the arguments.
+        # recreate the args object because inputcells may have been changed
+        new_args = args.unmatch_signature(self.signature, inputcells)
+        inputcells = self.parse_arguments(new_args, graph)
+        annotator = self.bookkeeper.annotator
+        result = annotator.recursivecall(graph, whence, inputcells)
+        signature = getattr(self.pyobj, '_signature_', None)
+        if signature:
+            sigresult = enforce_signature_return(self, signature[1], result)
+            if sigresult is not None:
+                annotator.addpendingblock(
+                    graph, graph.returnblock, [sigresult])
+                result = sigresult
+        # Some specializations may break the invariant of returning
+        # annotations that are always more general than the previous time.
+        # We restore it here:
+        result = unionof(result, s_previous_result)
+        return result
+
+    def normalize_args(self, inputs_s):
+        """
+        Canonicalize argument annotations into the exact parameter
+        annotations of a specific specialized graph.
+
+        Note: this method has no return value but mutates its argument instead.
+        """
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
@@ -304,39 +321,9 @@
                 from rpython.annotator.signature import Sig
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
-            enforceargs(self, inputcells)  # can modify inputcells in-place
+            enforceargs(self, inputs_s)  # can modify inputs_s in-place
         if signature:
-            enforce_signature_args(self, signature[0], inputcells)  # mutates inputcells
-        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
-            return self.specializer(self, inputcells, op)
-        else:
-            return self.specializer(self, inputcells)
-
-    def pycall(self, whence, args, s_previous_result, op=None):
-        inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells, op)
-        if isinstance(result, FunctionGraph):
-            graph = result         # common case
-            annotator = self.bookkeeper.annotator
-            # if that graph has a different signature, we need to re-parse
-            # the arguments.
-            # recreate the args object because inputcells may have been changed
-            new_args = args.unmatch_signature(self.signature, inputcells)
-            inputcells = self.parse_arguments(new_args, graph)
-            result = annotator.recursivecall(graph, whence, inputcells)
-            signature = getattr(self.pyobj, '_signature_', None)
-            if signature:
-                sigresult = enforce_signature_return(self, signature[1], result)
-                if sigresult is not None:
-                    annotator.addpendingblock(
-                        graph, graph.returnblock, [sigresult])
-                    result = sigresult
-        # Some specializations may break the invariant of returning
-        # annotations that are always more general than the previous time.
-        # We restore it here:
-        from rpython.annotator.model import unionof
-        result = unionof(result, s_previous_result)
-        return result
+            enforce_signature_args(self, signature[0], inputs_s)  # mutates inputs_s
 
     def get_graph(self, args, op):
         inputs_s = self.parse_arguments(args)
@@ -405,6 +392,16 @@
 
             return s_sigs
 
+class MemoDesc(FunctionDesc):
+    def pycall(self, whence, args, s_previous_result, op=None):
+        inputcells = self.parse_arguments(args)
+        s_result = self.specialize(inputcells, op)
+        if isinstance(s_result, FunctionGraph):
+            s_result = s_result.getreturnvar().annotation
+        s_result = unionof(s_result, s_previous_result)
+        return s_result
+
+
 class MethodDesc(Desc):
     knowntype = types.MethodType
 
diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py
--- a/rpython/annotator/specialize.py
+++ b/rpython/annotator/specialize.py
@@ -3,11 +3,13 @@
 
 from rpython.tool.sourcetools import func_with_new_name
 from rpython.tool.algo.unionfind import UnionFind
-from rpython.flowspace.model import Block, Link, Variable, SpaceOperation
+from rpython.flowspace.model import Block, Link, Variable
 from rpython.flowspace.model import checkgraph
 from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.flowspace.argument import Signature
+from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
+from rpython.annotator.model import unionof
 
 def flatten_star_args(funcdesc, args_s):
     argnames, vararg, kwarg = funcdesc.signature
@@ -127,7 +129,6 @@
     def finish(self):
         if self.do_not_process:
             return
-        from rpython.annotator.model import unionof
         assert self.graph is None, "MemoTable already finished"
         # list of which argument positions can take more than one value
         example_args, example_value = self.table.iteritems().next()
@@ -246,34 +247,36 @@
             args_s.append(unionof(*values_s))
         annotator.addpendinggraph(self.graph, args_s)
 
+def all_values(s):
+    """Return the exhaustive list of possible values matching annotation `s`.
 
-def memo(funcdesc, arglist_s):
-    from rpython.annotator.model import SomePBC, SomeImpossibleValue, SomeBool
-    from rpython.annotator.model import unionof
+    Raises `AnnotatorError` if no such (reasonably small) finite list exists.
+    """
+    if s.is_constant():
+        return [s.const]
+    elif isinstance(s, SomePBC):
+        values = []
+        assert not s.can_be_None, "memo call: cannot mix None and PBCs"
+        for desc in s.descriptions:
+            if desc.pyobj is None:
+                raise annmodel.AnnotatorError(
+                    "memo call with a class or PBC that has no "
+                    "corresponding Python object (%r)" % (desc,))
+            values.append(desc.pyobj)
+        return values
+    elif isinstance(s, SomeImpossibleValue):
+        return []
+    elif isinstance(s, SomeBool):
+        return [False, True]
+    else:
+        raise annmodel.AnnotatorError("memo call: argument must be a class "
+                                        "or a frozen PBC, got %r" % (s,))
+
+def memo(funcdesc, args_s):
     # call the function now, and collect possible results
-    argvalues = []
-    for s in arglist_s:
-        if s.is_constant():
-            values = [s.const]
-        elif isinstance(s, SomePBC):
-            values = []
-            assert not s.can_be_None, "memo call: cannot mix None and PBCs"
-            for desc in s.descriptions:
-                if desc.pyobj is None:
-                    raise annmodel.AnnotatorError(
-                        "memo call with a class or PBC that has no "
-                        "corresponding Python object (%r)" % (desc,))
-                values.append(desc.pyobj)
-        elif isinstance(s, SomeImpossibleValue):
-            return s    # we will probably get more possible args later
-        elif isinstance(s, SomeBool):
-            values = [False, True]
-        else:
-            raise annmodel.AnnotatorError("memo call: argument must be a class "
-                                          "or a frozen PBC, got %r" % (s,))
-        argvalues.append(values)
+
     # the list of all possible tuples of arguments to give to the memo function
-    possiblevalues = cartesian_product(argvalues)
+    possiblevalues = cartesian_product([all_values(s_arg) for s_arg in args_s])
 
     # a MemoTable factory -- one MemoTable per family of arguments that can
     # be called together, merged via a UnionFind.
diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py
--- a/rpython/annotator/test/test_model.py
+++ b/rpython/annotator/test/test_model.py
@@ -192,6 +192,20 @@
     assert union(union(s1, s2), s3) == union(s1, union(s2, s3))
 
 
+ at pytest.mark.xfail
+ at given(st_annotation, st_annotation)
+def test_generalize_isinstance(annotator, s1, s2):
+    try:
+        s_12 = union(s1, s2)
+    except UnionError:
+        assume(False)
+    assume(s1 != s_ImpossibleValue)
+    from rpython.annotator.unaryop import s_isinstance
+    s_int = annotator.bookkeeper.immutablevalue(int)
+    s_res_12 = s_isinstance(annotator, s_12, s_int, [])
+    s_res_1 = s_isinstance(annotator, s1, s_int, [])
+    assert s_res_12.contains(s_res_1)
+
 def compile_function(function, annotation=[]):
     t = TranslationContext()
     t.buildannotator().build_types(function, annotation)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -188,6 +188,10 @@
                "When true, enable the use of tagged pointers. "
                "If false, use normal boxing",
                default=False),
+    BoolOption("keepgoing",
+               "Continue annotating when errors are encountered, and report "
+               "them all at the end of the annotation phase",
+               default=False, cmdline="--keepgoing"),
     BoolOption("lldebug",
                "If true, makes an lldebug build", default=False,
                cmdline="--lldebug"),
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -268,12 +268,15 @@
         """
         mc = InstrBuilder(self.cpu.cpuinfo.arch_version)
         #
-        self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only)
+        # We don't save/restore r4; instead the return value (if any)
+        # will be stored there.
+        self._push_all_regs_to_jitframe(mc, [r.r4], self.cpu.supports_floats, callee_only)
         ## args are in their respective positions
         mc.PUSH([r.ip.value, r.lr.value])
         mc.BLX(r.r4.value)
+        mc.MOV_rr(r.r4.value, r.r0.value)
         self._reload_frame_if_necessary(mc)
-        self._pop_all_regs_from_jitframe(mc, [], supports_floats,
+        self._pop_all_regs_from_jitframe(mc, [r.r4], supports_floats,
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -357,7 +357,13 @@
         return fcond
 
     def emit_op_cond_call(self, op, arglocs, regalloc, fcond):
-        [call_loc] = arglocs
+        call_loc = arglocs[0]
+        if len(arglocs) == 2:
+            res_loc = arglocs[1]     # cond_call_value
+        else:
+            res_loc = None           # cond_call
+        # useless to list res_loc in the gcmap, because if the call is
+        # done it means res_loc was initially NULL
         gcmap = regalloc.get_gcmap([call_loc])
 
         assert call_loc is r.r4
@@ -378,8 +384,13 @@
                 floats = True
         cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
         self.mc.BL(cond_call_adr)
+        # if this is a COND_CALL_VALUE, we need to move the result in place
+        # from its current location (which is, unusually, in r4: see
+        # cond_call_slowpath)
+        if res_loc is not None and res_loc is not r.r4:
+            self.mc.MOV_rr(res_loc.value, r.r4.value)
+        #
         self.pop_gcmap(self.mc)
-        # never any result value
         cond = c.get_opposite_of(self.guard_success_cc)
         self.guard_success_cc = c.cond_none
         pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
@@ -389,6 +400,9 @@
         self.previous_cond_call_jcond = jmp_adr, cond
         return fcond
 
+    emit_op_cond_call_value_i = emit_op_cond_call
+    emit_op_cond_call_value_r = emit_op_cond_call
+
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         target_token = op.getdescr()
         assert isinstance(target_token, TargetToken)
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -1004,7 +1004,6 @@
     def prepare_op_cond_call(self, op, fcond):
         # XXX don't force the arguments to be loaded in specific
         # locations before knowing if we can take the fast path
-        # XXX add cond_call_value support
         assert 2 <= op.numargs() <= 4 + 2
         tmpreg = self.get_scratch_reg(INT, selected_reg=r.r4)
         v = op.getarg(1)
@@ -1017,8 +1016,33 @@
             arg = op.getarg(i)
             self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg)
             args_so_far.append(arg)
-        self.load_condition_into_cc(op.getarg(0))
-        return [tmpreg]
+
+        if op.type == 'v':
+            # a plain COND_CALL.  Calls the function when args[0] is
+            # true.  Often used just after a comparison operation.
+            self.load_condition_into_cc(op.getarg(0))
+            return [tmpreg]
+        else:
+            # COND_CALL_VALUE_I/R.  Calls the function when args[0]
+            # is equal to 0 or NULL.  Returns the result from the
+            # function call if done, or args[0] if it was not 0/NULL.
+            # Implemented by forcing the result to live in the same
+            # register as args[0], and overwriting it if we really do
+            # the call.
+
+            # Load the register for the result.  Possibly reuse 'args[0]'.
+            # But the old value of args[0], if it survives, is first
+            # spilled away.  We can't overwrite any of op.args[2:] here.
+            args = op.getarglist()
+            resloc = self.rm.force_result_in_reg(op, args[0],
+                                                 forbidden_vars=args[2:])
+            # Test the register for the result.
+            self.assembler.mc.CMP_ri(resloc.value, 0)
+            self.assembler.guard_success_cc = c.EQ
+            return [tmpreg, resloc]
+
+    prepare_op_cond_call_value_i = prepare_op_cond_call
+    prepare_op_cond_call_value_r = prepare_op_cond_call
 
     def prepare_op_force_token(self, op, fcond):
         # XXX for now we return a regular reg
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -15,11 +15,12 @@
 from rpython.rtyper.llinterp import LLInterpreter, LLException
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper import rclass
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong, intmask
-from rpython.rlib.objectmodel import Symbolic
+from rpython.rlib.objectmodel import Symbolic, compute_hash
 
 class LLAsmInfo(object):
     def __init__(self, lltrace):
@@ -326,7 +327,6 @@
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
     supports_guard_gc_type = True
-    supports_cond_call_value = True
     translate_support_code = False
     is_llgraph = True
     vector_ext = VectorExt()
@@ -789,6 +789,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_string_contents(src, dst, srcstart, dststart, length)
 
+    def bh_strhash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlstr(lls._as_ptr()))
+
     def bh_newunicode(self, length):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(rstr.UNICODE, length,
@@ -811,6 +815,10 @@
         assert 0 <= dststart <= dststart + length <= len(dst.chars)
         rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
 
+    def bh_unicodehash(self, s):
+        lls = s._obj.container
+        return compute_hash(hlunicode(lls._as_ptr()))
+
     def bh_new(self, sizedescr):
         return lltype.cast_opaque_ptr(llmemory.GCREF,
                                       lltype.malloc(sizedescr.S, zero=True))
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -3,8 +3,9 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
+from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -663,6 +664,14 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         return len(u.chars)
 
+    def bh_strhash(self, string):
+        s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
+        return compute_hash(hlstr(s))
+
+    def bh_unicodehash(self, string):
+        u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
+        return compute_hash(hlunicode(u))
+
     def bh_strgetitem(self, string, index):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return ord(s.chars[index])
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -9,9 +9,9 @@
 from rpython.jit.metainterp.typesystem import rd_eq, rd_hash
 from rpython.jit.codewriter import heaptracker
 from rpython.jit.backend.llsupport.symbolic import (WORD,
-        get_array_token)
+        get_field_token, get_array_token)
 from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,\
-     FLAG_POINTER, CallDescr
+     FLAG_POINTER
 from rpython.jit.metainterp.history import JitCellToken
 from rpython.jit.backend.llsupport.descr import (unpack_arraydescr,
         unpack_fielddescr, unpack_interiorfielddescr)
@@ -262,6 +262,18 @@
                                                  self.cpu.translate_support_code)
             self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
                                          WORD, 1, ofs_length, NOT_SIGNED)
+        elif opnum == rop.STRHASH:
+            offset, size = get_field_token(rstr.STR,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
+        elif opnum == rop.UNICODEHASH:
+            offset, size = get_field_token(rstr.UNICODE,
+                                        'hash', self.cpu.translate_support_code)
+            assert size == WORD
+            self.emit_gc_load_or_indexed(op, op.getarg(0), ConstInt(0),
+                                         WORD, 1, offset, sign=True)
         elif opnum == rop.STRGETITEM:
             basesize, itemsize, ofs_length = get_array_token(rstr.STR,
                                                  self.cpu.translate_support_code)
@@ -347,9 +359,7 @@
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:
                     self.consider_setarrayitem_gc(op)
-            # ---------- calls -----------
-            if OpHelpers.is_plain_call(op.getopnum()):
-                self.expand_call_shortcut(op)
+            # ---------- call assembler -----------
             if OpHelpers.is_call_assembler(op.getopnum()):
                 self.handle_call_assembler(op)
                 continue
@@ -595,33 +605,6 @@
         self.emit_gc_store_or_indexed(None, ptr, ConstInt(0), value,
                                       size, 1, ofs)
 
-    def expand_call_shortcut(self, op):
-        if not self.cpu.supports_cond_call_value:
-            return
-        descr = op.getdescr()

From pypy.commits at gmail.com  Wed Nov 30 10:43:03 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:43:03 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: isinstance(X,
 Y) is of course completely different than issubclass(type(X), Y)
Message-ID: <583ef387.0b561c0a.37b32.028e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88777:0ffbab13b5cd
Date: 2016-11-30 16:42 +0100
http://bitbucket.org/pypy/pypy/changeset/0ffbab13b5cd/

Log:	isinstance(X, Y) is of course completely different than
	issubclass(type(X), Y)

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -112,24 +112,30 @@
 
 
 class SimpleNamespace:
+    """A simple attribute-based namespace.
+
+SimpleNamespace(**kwargs)"""
+
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
 
-    def __repr__(self, recurse=set()):
+    def __repr__(self):
         ident = id(self)
-        if ident in recurse:
+        if ident in sns_recurse:
             return "namespace(...)"
-        recurse.add(ident)
+        sns_recurse.add(ident)
         try:
             pairs = ('%s=%r' % item for item in sorted(self.__dict__.items()))
             return "namespace(%s)" % ', '.join(pairs)
         finally:
-            recurse.remove(ident)
+            sns_recurse.discard(ident)
 
     def __eq__(self, other):
-        return (isinstance(other, SimpleNamespace) and
+        return (issubclass(type(other), SimpleNamespace) and
                 self.__dict__ == other.__dict__)
 
+sns_recurse = set()
+
 # This class is not exposed in sys, but by the types module.
 SimpleNamespace.__module__ = 'types'
 
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -611,6 +611,13 @@
         assert ns != Other(z=4, x=1, w=3)
         assert (Foo(z=4, x=1, w=3) == Other(z=4, x=1, w=3)) is False
         assert Foo(z=4, x=1, w=3) != Other(z=4, x=1, w=3)
+        #
+        class Fake:
+            __class__ = SimpleNamespace
+        assert isinstance(Fake(), SimpleNamespace)
+        assert not issubclass(Fake, SimpleNamespace)
+        assert (Fake() == SimpleNamespace()) is False
+        assert SimpleNamespace() != Fake()
 
     def test_pickle_simplenamespace(self):
         import pickle, sys

From pypy.commits at gmail.com  Wed Nov 30 10:46:39 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 07:46:39 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: tweak SimpleNamespace.__eq__ and __ne__
Message-ID: <583ef45f.46bb1c0a.1c422.030b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88778:3cce9c2e561f
Date: 2016-11-30 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/3cce9c2e561f/

Log:	tweak SimpleNamespace.__eq__ and __ne__

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -131,8 +131,14 @@
             sns_recurse.discard(ident)
 
     def __eq__(self, other):
-        return (issubclass(type(other), SimpleNamespace) and
-                self.__dict__ == other.__dict__)
+        if issubclass(type(other), SimpleNamespace):
+            return self.__dict__ == other.__dict__
+        return NotImplemented
+
+    def __ne__(self, other):
+        if issubclass(type(other), SimpleNamespace):
+            return self.__dict__ != other.__dict__
+        return NotImplemented
 
 sns_recurse = set()
 

From pypy.commits at gmail.com  Wed Nov 30 11:10:20 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 08:10:20 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Test and fix for super() with no
 arguments if 'self' is in a cell var
Message-ID: <583ef9ec.820bc30a.f79f4.f50c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88779:955d91968eb0
Date: 2016-11-30 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/955d91968eb0/

Log:	Test and fix for super() with no arguments if 'self' is in a cell
	var

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -76,7 +76,13 @@
         raise oefmt(space.w_RuntimeError, "super(): no code object")
     if code.co_argcount == 0:
         raise oefmt(space.w_RuntimeError, "super(): no arguments")
-    w_obj = frame.locals_cells_stack_w[0]
+    args_to_copy = code._args_as_cellvars
+    for i in range(len(args_to_copy)):
+        if args_to_copy[i] == 0:
+            w_obj = frame._getcell(i).w_value
+            break
+    else:
+        w_obj = frame.locals_cells_stack_w[0]
     if not w_obj:
         raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
 
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -508,4 +508,11 @@
                 del __class__
                 super()
         raises(RuntimeError, X().f)
+        class X:
+            def f(self):
+                def g():
+                    print(self)    # make 'self' a closure inside 'f'
+                del self
+                super()
+        raises(RuntimeError, X().f)
         """

From pypy.commits at gmail.com  Wed Nov 30 11:20:11 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 08:20:11 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: A test for super() with no argument,
 which I don't expect to pass *at all* so far
Message-ID: <583efc3b.4673c20a.e67f3.02d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88780:ef0611f31503
Date: 2016-11-30 17:19 +0100
http://bitbucket.org/pypy/pypy/changeset/ef0611f31503/

Log:	A test for super() with no argument, which I don't expect to pass
	*at all* so far

diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -320,3 +320,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_super_no_args(self):
+        def main():
+            class A(object):
+                def m(self, x):
+                    return x + 1
+            class B(A):
+                def m(self, x):
+                    return super().m(x)
+            i = 0
+            while i < 300:
+                i = B().m(i)
+            return i
+
+        log = self.run(main, [])
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i78 = int_lt(i72, 300)
+            guard_true(i78, descr=...)
+            guard_not_invalidated(descr=...)
+            i79 = force_token()
+            i80 = force_token()
+            i81 = int_add(i72, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)

From pypy.commits at gmail.com  Wed Nov 30 13:06:32 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 10:06:32 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: Try to re-add some JIT-friendliness
Message-ID: <583f1528.113cc20a.8d5cb.231e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88781:4622e566448f
Date: 2016-11-30 18:20 +0100
http://bitbucket.org/pypy/pypy/changeset/4622e566448f/

Log:	Try to re-add some JIT-friendliness

diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -1,3 +1,4 @@
+from rpython.rlib import jit
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -67,35 +68,43 @@
         # fallback to object.__getattribute__()
         return space.call_function(object_getattribute(space), self, w_name)
 
-def _super_from_frame(space, frame):
-    """super() without args -- fill in from __class__ and first local
-    variable on the stack.
-    """
-    code = frame.pycode
-    if not code:
-        raise oefmt(space.w_RuntimeError, "super(): no code object")
+ at jit.elidable
+def _get_self_location(space, code):
     if code.co_argcount == 0:
         raise oefmt(space.w_RuntimeError, "super(): no arguments")
     args_to_copy = code._args_as_cellvars
     for i in range(len(args_to_copy)):
         if args_to_copy[i] == 0:
-            w_obj = frame._getcell(i).w_value
+            self_cell = i
             break
     else:
-        w_obj = frame.locals_cells_stack_w[0]
-    if not w_obj:
-        raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
+        self_cell = -1
 
     for index, name in enumerate(code.co_freevars):
         if name == '__class__':
             break
     else:
         raise oefmt(space.w_RuntimeError, "super(): __class__ cell not found")
+    class_cell = len(code.co_cellvars) + index
+    return self_cell, class_cell
+
+def _super_from_frame(space, frame):
+    """super() without args -- fill in from __class__ and first local
+    variable on the stack.
+    """
+    if frame is None:
+        raise oefmt(space.w_RuntimeError, "super(): no frame object")
+    self_cell, class_cell = _get_self_location(space, frame.getcode())
+    if self_cell < 0:
+        w_obj = frame.locals_cells_stack_w[0]
+    else:
+        w_obj = frame._getcell(self_cell).w_value
+    if not w_obj:
+        raise oefmt(space.w_RuntimeError, "super(): arg[0] deleted")
+
     # a kind of LOAD_DEREF
-    cell = frame._getcell(len(code.co_cellvars) + index)
-    try:
-        w_starttype = cell.get()
-    except ValueError:
+    w_starttype = frame._getcell(class_cell).w_value
+    if w_starttype is None:
         raise oefmt(space.w_RuntimeError, "super(): empty __class__ cell")
     return w_starttype, w_obj
 

From pypy.commits at gmail.com  Wed Nov 30 13:06:34 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 10:06:34 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: add a test
Message-ID: <583f152a.6737c20a.58d18.4502@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88782:d05f950938d4
Date: 2016-11-30 18:24 +0100
http://bitbucket.org/pypy/pypy/changeset/d05f950938d4/

Log:	add a test

diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -515,4 +515,8 @@
                 del self
                 super()
         raises(RuntimeError, X().f)
+        class X:
+            def f(*args):
+                super()
+        raises(RuntimeError, X().f)
         """

From pypy.commits at gmail.com  Wed Nov 30 13:06:36 2016
From: pypy.commits at gmail.com (arigo)
Date: Wed, 30 Nov 2016 10:06:36 -0800 (PST)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <583f152c.c220c20a.7e1c1.386f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r88783:72ca1ab01fd4
Date: 2016-11-30 18:16 +0000
http://bitbucket.org/pypy/pypy/changeset/72ca1ab01fd4/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -340,9 +340,16 @@
             i78 = int_lt(i72, 300)
             guard_true(i78, descr=...)
             guard_not_invalidated(descr=...)
-            i79 = force_token()
-            i80 = force_token()
+            p1 = force_token()
+            p65 = force_token()
+            p3 = force_token()
             i81 = int_add(i72, 1)
-            --TICK--
+
+            # can't use TICK here, because of the extra setfield_gc
+            ticker0 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value .*>)
+            setfield_gc(p0, p65, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+
             jump(..., descr=...)
         """)